2024-04-26T15:46:25.848 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-26T15:46:25.848 DEBUG:teuthology.run:Teuthology command: teuthology --name phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi --archive /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817 --description orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} --owner scheduled_phlogistonjohn@teuthology --verbose -- /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817/orig.config.yaml 2024-04-26T15:46:25.895 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T15:46:25.985 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817 branch: wip-phlogistonjohn-testing-2024-04-25-1915 description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: jmulligan@redhat.com first_in_suite: false job_id: '7674817' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-phlogistonjohn-testing-2024-04-25-1915 ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 workunit: branch: wip-phlogistonjohn-testing-2024-04-25-1915 sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 owner: scheduled_phlogistonjohn@teuthology priority: 89 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 4852 sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 sleep_before_teardown: 0 subset: 1/30 suite: orch:cephadm suite_branch: wip-phlogistonjohn-testing-2024-04-25-1915 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_a430f9ed84de3008ca5b632b1ed304bac0e753f6/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 targets: smithi059.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFVJcwblfPLN9tgjmFFh+3o/rieQnpkT9ya3MoVuf0keS/HFhkWfmKmHD2GMT1ipPS8ldbV/6QvNluD1QKAUlG0= smithi064.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP2vydMHmlTV/e4Mdcnz1q5GRy4z/5gVzfYCuSx3mMdEwaP2Y9RqoqKPZCloEUWumDfo1JvnMWLg48JG/C3hgwM= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-26_15:06:39 tube: smithi user: phlogistonjohn verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-26T15:46:25.985 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_a430f9ed84de3008ca5b632b1ed304bac0e753f6/qa; will attempt to use it 2024-04-26T15:46:25.986 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_a430f9ed84de3008ca5b632b1ed304bac0e753f6/qa/tasks 2024-04-26T15:46:25.986 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-26T15:46:25.987 INFO:teuthology.task.internal:Checking packages... 2024-04-26T15:46:26.012 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'a430f9ed84de3008ca5b632b1ed304bac0e753f6' 2024-04-26T15:46:26.012 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T15:46:26.012 INFO:teuthology.packaging:ref: None 2024-04-26T15:46:26.012 INFO:teuthology.packaging:tag: None 2024-04-26T15:46:26.012 INFO:teuthology.packaging:branch: wip-phlogistonjohn-testing-2024-04-25-1915 2024-04-26T15:46:26.012 INFO:teuthology.packaging:sha1: a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T15:46:26.013 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-phlogistonjohn-testing-2024-04-25-1915 2024-04-26T15:46:26.248 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3305.ga430f9ed 2024-04-26T15:46:26.250 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-26T15:46:26.262 INFO:teuthology.task.internal:no buildpackages task found 2024-04-26T15:46:26.262 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-26T15:46:26.287 INFO:teuthology.task.internal:Saving configuration 2024-04-26T15:46:26.305 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-26T15:46:26.316 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T15:46:26.341 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi059.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 15:41:24.415640', 'locked_by': 'scheduled_phlogistonjohn@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFVJcwblfPLN9tgjmFFh+3o/rieQnpkT9ya3MoVuf0keS/HFhkWfmKmHD2GMT1ipPS8ldbV/6QvNluD1QKAUlG0='} 2024-04-26T15:46:26.367 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi064.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 15:41:24.417563', 'locked_by': 'scheduled_phlogistonjohn@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP2vydMHmlTV/e4Mdcnz1q5GRy4z/5gVzfYCuSx3mMdEwaP2Y9RqoqKPZCloEUWumDfo1JvnMWLg48JG/C3hgwM='} 2024-04-26T15:46:26.367 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-26T15:46:26.378 INFO:teuthology.task.internal:roles: ubuntu@smithi059.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-26T15:46:26.379 INFO:teuthology.task.internal:roles: ubuntu@smithi064.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-26T15:46:26.379 INFO:teuthology.run_tasks:Running task console_log... 2024-04-26T15:46:26.493 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f14136480d0>, signals=[15]) 2024-04-26T15:46:26.493 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-26T15:46:26.505 INFO:teuthology.task.internal:Opening connections... 2024-04-26T15:46:26.505 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi059.front.sepia.ceph.com 2024-04-26T15:46:26.507 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi059.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T15:46:26.585 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi064.front.sepia.ceph.com 2024-04-26T15:46:26.586 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T15:46:26.663 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-26T15:46:26.676 DEBUG:teuthology.orchestra.run.smithi059:> uname -m 2024-04-26T15:46:26.694 INFO:teuthology.orchestra.run.smithi059.stdout:x86_64 2024-04-26T15:46:26.694 DEBUG:teuthology.orchestra.run.smithi059:> cat /etc/os-release 2024-04-26T15:46:26.754 INFO:teuthology.orchestra.run.smithi059.stdout:NAME="CentOS Stream" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:VERSION="9" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:ID="centos" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:ID_LIKE="rhel fedora" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:VERSION_ID="9" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:PLATFORM_ID="platform:el9" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:ANSI_COLOR="0;31" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:LOGO="fedora-logo-icon" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:HOME_URL="https://centos.org/" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T15:46:26.755 INFO:teuthology.orchestra.run.smithi059.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T15:46:26.756 INFO:teuthology.lock.ops:Updating smithi059.front.sepia.ceph.com on lock server 2024-04-26T15:46:26.773 DEBUG:teuthology.orchestra.run.smithi064:> uname -m 2024-04-26T15:46:26.792 INFO:teuthology.orchestra.run.smithi064.stdout:x86_64 2024-04-26T15:46:26.792 DEBUG:teuthology.orchestra.run.smithi064:> cat /etc/os-release 2024-04-26T15:46:26.851 INFO:teuthology.orchestra.run.smithi064.stdout:NAME="CentOS Stream" 2024-04-26T15:46:26.851 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION="9" 2024-04-26T15:46:26.851 INFO:teuthology.orchestra.run.smithi064.stdout:ID="centos" 2024-04-26T15:46:26.851 INFO:teuthology.orchestra.run.smithi064.stdout:ID_LIKE="rhel fedora" 2024-04-26T15:46:26.851 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION_ID="9" 2024-04-26T15:46:26.851 INFO:teuthology.orchestra.run.smithi064.stdout:PLATFORM_ID="platform:el9" 2024-04-26T15:46:26.851 INFO:teuthology.orchestra.run.smithi064.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T15:46:26.851 INFO:teuthology.orchestra.run.smithi064.stdout:ANSI_COLOR="0;31" 2024-04-26T15:46:26.852 INFO:teuthology.orchestra.run.smithi064.stdout:LOGO="fedora-logo-icon" 2024-04-26T15:46:26.852 INFO:teuthology.orchestra.run.smithi064.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T15:46:26.852 INFO:teuthology.orchestra.run.smithi064.stdout:HOME_URL="https://centos.org/" 2024-04-26T15:46:26.852 INFO:teuthology.orchestra.run.smithi064.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T15:46:26.852 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T15:46:26.852 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T15:46:26.852 INFO:teuthology.lock.ops:Updating smithi064.front.sepia.ceph.com on lock server 2024-04-26T15:46:26.872 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-26T15:46:26.885 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-26T15:46:26.897 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-26T15:46:26.898 DEBUG:teuthology.orchestra.run.smithi059:> test '!' -e /home/ubuntu/cephtest 2024-04-26T15:46:26.900 DEBUG:teuthology.orchestra.run.smithi064:> test '!' -e /home/ubuntu/cephtest 2024-04-26T15:46:26.916 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-26T15:46:26.927 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-26T15:46:26.928 DEBUG:teuthology.orchestra.run.smithi059:> test -z $(ls -A /var/lib/ceph) 2024-04-26T15:46:26.957 DEBUG:teuthology.orchestra.run.smithi064:> test -z $(ls -A /var/lib/ceph) 2024-04-26T15:46:26.986 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-26T15:46:27.048 INFO:teuthology.run_tasks:Running task kernel... 2024-04-26T15:46:27.065 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-04-26T15:46:27.065 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-04-26T15:46:27.065 DEBUG:teuthology.orchestra.run.smithi059:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T15:46:27.066 DEBUG:teuthology.orchestra.run.smithi064:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T15:46:27.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:46:27.083 DEBUG:teuthology.orchestra.run.smithi059:> uname -r 2024-04-26T15:46:27.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:46:27.084 DEBUG:teuthology.orchestra.run.smithi064:> uname -r 2024-04-26T15:46:27.141 INFO:teuthology.orchestra.run.smithi059.stdout:5.14.0-437.el9.x86_64 2024-04-26T15:46:27.142 INFO:teuthology.task.kernel:Running kernel on smithi059: 5.14.0-437.el9.x86_64 2024-04-26T15:46:27.142 DEBUG:teuthology.orchestra.run.smithi059:> sudo yum install -y kernel 2024-04-26T15:46:27.143 INFO:teuthology.orchestra.run.smithi064.stdout:5.14.0-437.el9.x86_64 2024-04-26T15:46:27.143 INFO:teuthology.task.kernel:Running kernel on smithi064: 5.14.0-437.el9.x86_64 2024-04-26T15:46:27.143 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum install -y kernel 2024-04-26T15:46:29.338 INFO:teuthology.orchestra.run.smithi059.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.0 MB 00:00 2024-04-26T15:46:29.580 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - BaseOS 9.5 MB/s | 8.0 MB 00:00 2024-04-26T15:46:31.106 INFO:teuthology.orchestra.run.smithi059.stdout:CentOS Stream 9 - AppStream 29 MB/s | 19 MB 00:00 2024-04-26T15:46:31.614 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - AppStream 20 MB/s | 19 MB 00:00 2024-04-26T15:46:35.356 INFO:teuthology.orchestra.run.smithi059.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.2 MB 00:00 2024-04-26T15:46:35.990 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - CRB 9.6 MB/s | 6.2 MB 00:00 2024-04-26T15:46:36.821 INFO:teuthology.orchestra.run.smithi059.stdout:CentOS Stream 9 - Extras packages 62 kB/s | 16 kB 00:00 2024-04-26T15:46:37.680 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - Extras packages 33 kB/s | 16 kB 00:00 2024-04-26T15:46:38.266 INFO:teuthology.orchestra.run.smithi059.stdout:Extra Packages for Enterprise Linux 16 MB/s | 21 MB 00:01 2024-04-26T15:46:42.564 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 4.4 MB/s | 21 MB 00:04 2024-04-26T15:46:43.447 INFO:teuthology.orchestra.run.smithi059.stdout:lab-extras 47 kB/s | 1.7 kB 00:00 2024-04-26T15:46:45.379 INFO:teuthology.orchestra.run.smithi059.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T15:46:45.379 INFO:teuthology.orchestra.run.smithi059.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T15:46:45.461 INFO:teuthology.orchestra.run.smithi059.stdout:Dependencies resolved. 2024-04-26T15:46:45.462 INFO:teuthology.orchestra.run.smithi059.stdout:Nothing to do. 2024-04-26T15:46:45.462 INFO:teuthology.orchestra.run.smithi059.stdout:Complete! 2024-04-26T15:46:45.564 DEBUG:teuthology.orchestra.run.smithi059:> echo no | sudo yum reinstall kernel || true 2024-04-26T15:46:46.090 INFO:teuthology.orchestra.run.smithi059.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 03:46:43 PM UTC. 2024-04-26T15:46:46.214 INFO:teuthology.orchestra.run.smithi059.stdout:Dependencies resolved. 2024-04-26T15:46:46.230 INFO:teuthology.orchestra.run.smithi059.stdout:================================================================================ 2024-04-26T15:46:46.230 INFO:teuthology.orchestra.run.smithi059.stdout: Package Architecture Version Repository Size 2024-04-26T15:46:46.230 INFO:teuthology.orchestra.run.smithi059.stdout:================================================================================ 2024-04-26T15:46:46.230 INFO:teuthology.orchestra.run.smithi059.stdout:Reinstalling: 2024-04-26T15:46:46.230 INFO:teuthology.orchestra.run.smithi059.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T15:46:46.230 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:46:46.230 INFO:teuthology.orchestra.run.smithi059.stdout:Transaction Summary 2024-04-26T15:46:46.230 INFO:teuthology.orchestra.run.smithi059.stdout:================================================================================ 2024-04-26T15:46:46.231 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:46:46.231 INFO:teuthology.orchestra.run.smithi059.stdout:Total download size: 255 k 2024-04-26T15:46:46.231 INFO:teuthology.orchestra.run.smithi059.stdout:Installed size: 0 2024-04-26T15:46:46.234 INFO:teuthology.orchestra.run.smithi059.stderr:Operation aborted. 2024-04-26T15:46:46.289 INFO:teuthology.orchestra.run.smithi059.stdout:Is this ok [y/N]: 2024-04-26T15:46:46.290 DEBUG:teuthology.orchestra.run.smithi059:> sudo yum reinstall -y kernel || true 2024-04-26T15:46:46.808 INFO:teuthology.orchestra.run.smithi059.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 03:46:43 PM UTC. 2024-04-26T15:46:46.928 INFO:teuthology.orchestra.run.smithi059.stdout:Dependencies resolved. 2024-04-26T15:46:46.929 INFO:teuthology.orchestra.run.smithi059.stdout:================================================================================ 2024-04-26T15:46:46.930 INFO:teuthology.orchestra.run.smithi059.stdout: Package Architecture Version Repository Size 2024-04-26T15:46:46.930 INFO:teuthology.orchestra.run.smithi059.stdout:================================================================================ 2024-04-26T15:46:46.930 INFO:teuthology.orchestra.run.smithi059.stdout:Reinstalling: 2024-04-26T15:46:46.930 INFO:teuthology.orchestra.run.smithi059.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T15:46:46.930 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:46:46.930 INFO:teuthology.orchestra.run.smithi059.stdout:Transaction Summary 2024-04-26T15:46:46.930 INFO:teuthology.orchestra.run.smithi059.stdout:================================================================================ 2024-04-26T15:46:46.931 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:46:46.931 INFO:teuthology.orchestra.run.smithi059.stdout:Total download size: 255 k 2024-04-26T15:46:46.931 INFO:teuthology.orchestra.run.smithi059.stdout:Installed size: 0 2024-04-26T15:46:46.931 INFO:teuthology.orchestra.run.smithi059.stdout:Downloading Packages: 2024-04-26T15:46:47.183 INFO:teuthology.orchestra.run.smithi059.stdout:kernel-5.14.0-437.el9.x86_64.rpm 2.0 MB/s | 255 kB 00:00 2024-04-26T15:46:47.183 INFO:teuthology.orchestra.run.smithi059.stdout:-------------------------------------------------------------------------------- 2024-04-26T15:46:47.183 INFO:teuthology.orchestra.run.smithi059.stdout:Total 1.0 MB/s | 255 kB 00:00 2024-04-26T15:46:47.200 INFO:teuthology.orchestra.run.smithi059.stdout:Running transaction check 2024-04-26T15:46:47.383 INFO:teuthology.orchestra.run.smithi059.stdout:Transaction check succeeded. 2024-04-26T15:46:47.383 INFO:teuthology.orchestra.run.smithi059.stdout:Running transaction test 2024-04-26T15:46:47.388 INFO:teuthology.orchestra.run.smithi059.stdout:Transaction test succeeded. 2024-04-26T15:46:47.388 INFO:teuthology.orchestra.run.smithi059.stdout:Running transaction 2024-04-26T15:46:47.552 INFO:teuthology.orchestra.run.smithi059.stdout: Preparing : 1/1 2024-04-26T15:46:47.626 INFO:teuthology.orchestra.run.smithi059.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T15:46:47.626 INFO:teuthology.orchestra.run.smithi059.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:46:47.714 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 48 kB/s | 1.7 kB 00:00 2024-04-26T15:46:48.287 INFO:teuthology.orchestra.run.smithi059.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:46:48.287 INFO:teuthology.orchestra.run.smithi059.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T15:46:49.025 INFO:teuthology.orchestra.run.smithi059.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:46:49.025 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:46:49.025 INFO:teuthology.orchestra.run.smithi059.stdout:Reinstalled: 2024-04-26T15:46:49.025 INFO:teuthology.orchestra.run.smithi059.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T15:46:49.026 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:46:49.026 INFO:teuthology.orchestra.run.smithi059.stdout:Complete! 2024-04-26T15:46:49.206 DEBUG:teuthology.orchestra.run.smithi059:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T15:46:49.255 INFO:teuthology.orchestra.run.smithi059.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T15:46:49.255 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T15:46:49.256 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T15:46:49.256 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-26T15:46:49.256 DEBUG:teuthology.orchestra.run.smithi059:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T15:46:49.296 INFO:teuthology.orchestra.run.smithi059.stdout:ttyS1 2024-04-26T15:46:49.329 DEBUG:teuthology.parallel:result is None 2024-04-26T15:46:49.782 INFO:teuthology.orchestra.run.smithi064.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T15:46:49.782 INFO:teuthology.orchestra.run.smithi064.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T15:46:49.876 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-04-26T15:46:49.877 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2024-04-26T15:46:49.877 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-04-26T15:46:49.979 DEBUG:teuthology.orchestra.run.smithi064:> echo no | sudo yum reinstall kernel || true 2024-04-26T15:46:50.500 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 03:46:47 PM UTC. 2024-04-26T15:46:50.624 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-04-26T15:46:50.643 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-04-26T15:46:50.643 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2024-04-26T15:46:50.643 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-04-26T15:46:50.643 INFO:teuthology.orchestra.run.smithi064.stdout:Reinstalling: 2024-04-26T15:46:50.644 INFO:teuthology.orchestra.run.smithi064.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T15:46:50.644 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-04-26T15:46:50.644 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-04-26T15:46:50.644 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-04-26T15:46:50.644 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-04-26T15:46:50.644 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 255 k 2024-04-26T15:46:50.644 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 0 2024-04-26T15:46:50.645 INFO:teuthology.orchestra.run.smithi064.stderr:Operation aborted. 2024-04-26T15:46:50.701 INFO:teuthology.orchestra.run.smithi064.stdout:Is this ok [y/N]: 2024-04-26T15:46:50.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum reinstall -y kernel || true 2024-04-26T15:46:51.218 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:04 ago on Fri 26 Apr 2024 03:46:47 PM UTC. 2024-04-26T15:46:51.338 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-04-26T15:46:51.338 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout:Reinstalling: 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 255 k 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 0 2024-04-26T15:46:51.339 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-04-26T15:46:51.682 INFO:teuthology.orchestra.run.smithi064.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.5 MB/s | 255 kB 00:00 2024-04-26T15:46:51.683 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-04-26T15:46:51.683 INFO:teuthology.orchestra.run.smithi064.stdout:Total 740 kB/s | 255 kB 00:00 2024-04-26T15:46:51.699 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-04-26T15:46:51.886 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-04-26T15:46:51.886 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-04-26T15:46:51.891 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-04-26T15:46:51.892 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-04-26T15:46:52.030 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-04-26T15:46:52.100 INFO:teuthology.orchestra.run.smithi064.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T15:46:52.101 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:46:53.052 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:46:53.052 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T15:46:53.739 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T15:46:53.740 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-04-26T15:46:53.740 INFO:teuthology.orchestra.run.smithi064.stdout:Reinstalled: 2024-04-26T15:46:53.740 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T15:46:53.740 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-04-26T15:46:53.740 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-04-26T15:46:53.915 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T15:46:53.962 INFO:teuthology.orchestra.run.smithi064.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T15:46:53.963 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T15:46:53.963 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T15:46:53.963 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-26T15:46:53.963 DEBUG:teuthology.orchestra.run.smithi064:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T15:46:54.067 INFO:teuthology.orchestra.run.smithi064.stdout:ttyS1 2024-04-26T15:46:54.085 DEBUG:teuthology.parallel:result is None 2024-04-26T15:46:54.085 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-26T15:46:54.098 INFO:teuthology.task.internal:Creating test directory... 2024-04-26T15:46:54.098 DEBUG:teuthology.orchestra.run.smithi059:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T15:46:54.101 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T15:46:54.116 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-26T15:46:54.124 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-26T15:46:54.133 INFO:teuthology.task.internal:Creating archive directory... 2024-04-26T15:46:54.134 DEBUG:teuthology.orchestra.run.smithi059:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T15:46:54.159 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T15:46:54.190 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-26T15:46:54.197 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-26T15:46:54.197 DEBUG:teuthology.orchestra.run.smithi059:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T15:46:54.225 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T15:46:54.260 INFO:teuthology.orchestra.run.smithi059.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T15:46:54.270 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T15:46:54.271 INFO:teuthology.orchestra.run.smithi059.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T15:46:54.279 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T15:46:54.281 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-26T15:46:54.291 INFO:teuthology.task.internal:Configuring sudo... 2024-04-26T15:46:54.291 DEBUG:teuthology.orchestra.run.smithi059:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T15:46:54.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T15:46:54.341 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-26T15:46:54.350 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-26T15:46:54.351 DEBUG:teuthology.orchestra.run.smithi059:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T15:46:54.383 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T15:46:54.399 DEBUG:teuthology.orchestra.run.smithi059:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T15:46:54.483 DEBUG:teuthology.orchestra.run.smithi059:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T15:46:54.531 DEBUG:teuthology.orchestra.run.smithi059:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T15:46:54.615 DEBUG:teuthology.orchestra.run.smithi059:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T15:46:54.655 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-26T15:46:54.656 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T15:46:54.722 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T15:46:54.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T15:46:54.818 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T15:46:54.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T15:46:54.942 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-04-26T15:46:54.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T15:46:55.007 DEBUG:teuthology.orchestra.run.smithi059:> sudo service rsyslog restart 2024-04-26T15:46:55.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo service rsyslog restart 2024-04-26T15:46:55.048 INFO:teuthology.orchestra.run.smithi059.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T15:46:55.095 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T15:46:55.476 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-26T15:46:55.484 INFO:teuthology.task.internal:Starting timer... 2024-04-26T15:46:55.484 INFO:teuthology.run_tasks:Running task pcp... 2024-04-26T15:46:55.501 INFO:teuthology.run_tasks:Running task selinux... 2024-04-26T15:46:55.514 DEBUG:teuthology.orchestra.run.smithi059:> sudo service auditd rotate 2024-04-26T15:46:55.591 INFO:teuthology.orchestra.run.smithi059.stdout:Rotating logs: 2024-04-26T15:46:55.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo service auditd rotate 2024-04-26T15:46:55.655 INFO:teuthology.orchestra.run.smithi064.stdout:Rotating logs: 2024-04-26T15:46:55.657 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-26T15:46:55.657 DEBUG:teuthology.orchestra.run.smithi059:> /usr/sbin/getenforce 2024-04-26T15:46:55.676 INFO:teuthology.orchestra.run.smithi059.stdout:Permissive 2024-04-26T15:46:55.677 DEBUG:teuthology.orchestra.run.smithi064:> /usr/sbin/getenforce 2024-04-26T15:46:55.701 INFO:teuthology.orchestra.run.smithi064.stdout:Permissive 2024-04-26T15:46:55.702 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi059.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi064.front.sepia.ceph.com': 'permissive'} 2024-04-26T15:46:55.702 DEBUG:teuthology.orchestra.run.smithi059:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-26T15:46:55.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:46:55.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-26T15:46:55.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T15:46:55.770 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-26T15:46:55.770 DEBUG:teuthology.orchestra.run.smithi059:> sudo /usr/sbin/setenforce permissive 2024-04-26T15:46:55.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo /usr/sbin/setenforce permissive 2024-04-26T15:46:55.834 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-26T15:46:55.845 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-26T15:46:55.861 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-26T15:46:55.862 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi059.front.sepia.ceph.com,smithi064.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-26T15:50:31.279 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi059.front.sepia.ceph.com'), Remote(name='ubuntu@smithi064.front.sepia.ceph.com')] 2024-04-26T15:50:31.281 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi059.front.sepia.ceph.com' 2024-04-26T15:50:31.282 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi059.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T15:50:31.361 DEBUG:teuthology.orchestra.run.smithi059:> true 2024-04-26T15:50:31.439 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi059.front.sepia.ceph.com' 2024-04-26T15:50:31.439 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-04-26T15:50:31.440 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T15:50:31.516 DEBUG:teuthology.orchestra.run.smithi064:> true 2024-04-26T15:50:31.598 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-04-26T15:50:31.598 INFO:teuthology.run_tasks:Running task clock... 2024-04-26T15:50:31.609 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-26T15:50:31.609 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T15:50:31.609 DEBUG:teuthology.orchestra.run.smithi059:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T15:50:31.612 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T15:50:31.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T15:50:31.641 INFO:teuthology.orchestra.run.smithi059.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T15:50:31.656 INFO:teuthology.orchestra.run.smithi059.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T15:50:31.668 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T15:50:31.680 INFO:teuthology.orchestra.run.smithi059.stderr:sudo: ntpd: command not found 2024-04-26T15:50:31.682 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T15:50:31.692 INFO:teuthology.orchestra.run.smithi059.stdout:506 Cannot talk to daemon 2024-04-26T15:50:31.706 INFO:teuthology.orchestra.run.smithi059.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T15:50:31.706 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: ntpd: command not found 2024-04-26T15:50:31.717 INFO:teuthology.orchestra.run.smithi064.stdout:506 Cannot talk to daemon 2024-04-26T15:50:31.720 INFO:teuthology.orchestra.run.smithi059.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T15:50:31.731 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T15:50:31.745 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T15:50:31.776 INFO:teuthology.orchestra.run.smithi059.stderr:bash: line 1: ntpq: command not found 2024-04-26T15:50:31.781 INFO:teuthology.orchestra.run.smithi059.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T15:50:31.781 INFO:teuthology.orchestra.run.smithi059.stdout:=============================================================================== 2024-04-26T15:50:31.781 INFO:teuthology.orchestra.run.smithi059.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:50:31.781 INFO:teuthology.orchestra.run.smithi059.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:50:31.781 INFO:teuthology.orchestra.run.smithi059.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:50:31.781 INFO:teuthology.orchestra.run.smithi059.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:50:31.799 INFO:teuthology.orchestra.run.smithi064.stderr:bash: line 1: ntpq: command not found 2024-04-26T15:50:31.803 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T15:50:31.803 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2024-04-26T15:50:31.803 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:50:31.804 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:50:31.804 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:50:31.804 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T15:50:31.804 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-26T15:50:31.897 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': 'a430f9ed84de3008ca5b632b1ed304bac0e753f6'} 2024-04-26T15:50:31.897 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-26T15:50:31.897 INFO:tasks.cephadm:Cluster fsid is b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:50:31.898 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-26T15:50:31.898 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.59', 'mon.c': '[v2:172.21.15.59:3301,v1:172.21.15.59:6790]', 'mon.b': '172.21.15.64'} 2024-04-26T15:50:31.898 INFO:tasks.cephadm:First mon is mon.a on smithi059 2024-04-26T15:50:31.898 INFO:tasks.cephadm:First mgr is y 2024-04-26T15:50:31.898 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-26T15:50:31.898 DEBUG:teuthology.orchestra.run.smithi059:> sudo hostname $(hostname -s) 2024-04-26T15:50:31.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo hostname $(hostname -s) 2024-04-26T15:50:31.949 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-26T15:50:31.949 DEBUG:teuthology.orchestra.run.smithi059:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T15:50:32.209 INFO:teuthology.orchestra.run.smithi059.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 26 15:50 /home/ubuntu/cephtest/cephadm 2024-04-26T15:50:32.209 DEBUG:teuthology.orchestra.run.smithi064:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T15:50:32.380 INFO:teuthology.orchestra.run.smithi064.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 26 15:50 /home/ubuntu/cephtest/cephadm 2024-04-26T15:50:32.380 DEBUG:teuthology.orchestra.run.smithi059:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T15:50:32.396 DEBUG:teuthology.orchestra.run.smithi064:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T15:50:32.422 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-26T15:50:32.422 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-26T15:50:32.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-26T15:50:33.889 INFO:teuthology.orchestra.run.smithi064.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-26T15:50:34.036 INFO:teuthology.orchestra.run.smithi059.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-26T15:50:53.143 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-04-26T15:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-26T15:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-26T15:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: "repo_digests": [ 2024-04-26T15:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-26T15:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-26T15:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-04-26T15:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-04-26T15:50:53.434 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T15:50:53.434 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-26T15:50:53.434 INFO:teuthology.orchestra.run.smithi059.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-26T15:50:53.434 INFO:teuthology.orchestra.run.smithi059.stdout: "repo_digests": [ 2024-04-26T15:50:53.434 INFO:teuthology.orchestra.run.smithi059.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-26T15:50:53.434 INFO:teuthology.orchestra.run.smithi059.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-26T15:50:53.434 INFO:teuthology.orchestra.run.smithi059.stdout: ] 2024-04-26T15:50:53.435 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T15:50:53.452 DEBUG:teuthology.orchestra.run.smithi059:> sudo mkdir -p /etc/ceph 2024-04-26T15:50:53.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph 2024-04-26T15:50:53.534 DEBUG:teuthology.orchestra.run.smithi059:> sudo chmod 777 /etc/ceph 2024-04-26T15:50:53.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 777 /etc/ceph 2024-04-26T15:50:53.598 INFO:tasks.cephadm:Writing seed config... 2024-04-26T15:50:53.599 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-04-26T15:50:53.599 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-26T15:50:53.599 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-26T15:50:53.600 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-04-26T15:50:53.600 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-26T15:50:53.600 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-26T15:50:53.600 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-26T15:50:53.600 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-26T15:50:53.600 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-26T15:50:53.600 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-26T15:50:53.601 DEBUG:teuthology.orchestra.run.smithi059:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-26T15:50:53.616 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = b6607c98-03e4-11ef-bc93-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-26T15:50:53.617 DEBUG:teuthology.orchestra.run.smithi059:mon.a> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a.service 2024-04-26T15:50:53.659 DEBUG:teuthology.orchestra.run.smithi059:mgr.y> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y.service 2024-04-26T15:50:53.701 INFO:tasks.cephadm:Bootstrapping... 2024-04-26T15:50:53.701 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid b6607c98-03e4-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.59 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-26T15:50:53.900 INFO:teuthology.orchestra.run.smithi059.stderr:-------------------------------------------------------------------------------- 2024-04-26T15:50:53.900 INFO:teuthology.orchestra.run.smithi059.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'b6607c98-03e4-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.59', '--skip-admin-label'] 2024-04-26T15:50:53.923 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: 4.9.4-dev 2024-04-26T15:50:53.926 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying podman|docker is present... 2024-04-26T15:50:53.956 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: 4.9.4-dev 2024-04-26T15:50:53.958 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying lvm2 is present... 2024-04-26T15:50:53.958 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying time synchronization is in place... 2024-04-26T15:50:53.965 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T15:50:53.971 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: inactive 2024-04-26T15:50:53.978 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: enabled 2024-04-26T15:50:53.984 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: active 2024-04-26T15:50:53.984 INFO:teuthology.orchestra.run.smithi059.stderr:Unit chronyd.service is enabled and running 2024-04-26T15:50:53.984 INFO:teuthology.orchestra.run.smithi059.stderr:Repeating the final host check... 2024-04-26T15:50:54.007 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: 4.9.4-dev 2024-04-26T15:50:54.009 INFO:teuthology.orchestra.run.smithi059.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-26T15:50:54.009 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl is present 2024-04-26T15:50:54.009 INFO:teuthology.orchestra.run.smithi059.stderr:lvcreate is present 2024-04-26T15:50:54.016 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T15:50:54.024 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: inactive 2024-04-26T15:50:54.033 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: enabled 2024-04-26T15:50:54.040 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: active 2024-04-26T15:50:54.041 INFO:teuthology.orchestra.run.smithi059.stderr:Unit chronyd.service is enabled and running 2024-04-26T15:50:54.041 INFO:teuthology.orchestra.run.smithi059.stderr:Host looks OK 2024-04-26T15:50:54.041 INFO:teuthology.orchestra.run.smithi059.stderr:Cluster fsid: b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:50:54.041 INFO:teuthology.orchestra.run.smithi059.stderr:Acquiring lock 140539892023152 on /run/cephadm/b6607c98-03e4-11ef-bc93-c7b262605968.lock 2024-04-26T15:50:54.041 INFO:teuthology.orchestra.run.smithi059.stderr:Lock 140539892023152 acquired on /run/cephadm/b6607c98-03e4-11ef-bc93-c7b262605968.lock 2024-04-26T15:50:54.042 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying IP 172.21.15.59 port 3300 ... 2024-04-26T15:50:54.042 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying IP 172.21.15.59 port 6789 ... 2024-04-26T15:50:54.042 INFO:teuthology.orchestra.run.smithi059.stderr:Base mon IP is 172.21.15.59, final addrv is [v2:172.21.15.59:3300,v1:172.21.15.59:6789] 2024-04-26T15:50:54.044 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.59 metric 100 2024-04-26T15:50:54.045 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.59 metric 100 2024-04-26T15:50:54.048 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-26T15:50:54.048 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-04-26T15:50:54.048 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-04-26T15:50:54.051 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-26T15:50:54.051 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-26T15:50:54.051 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-26T15:50:54.052 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-04-26T15:50:54.052 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:192a/64 scope link noprefixroute 2024-04-26T15:50:54.052 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-26T15:50:54.052 INFO:teuthology.orchestra.run.smithi059.stderr:Mon IP `172.21.15.59` is in CIDR network `172.21.0.0/20` 2024-04-26T15:50:54.053 INFO:teuthology.orchestra.run.smithi059.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-26T15:50:54.053 INFO:teuthology.orchestra.run.smithi059.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-26T15:50:54.159 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-26T15:50:54.578 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: Getting image source signatures 2024-04-26T15:50:54.579 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-26T15:50:54.579 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-26T15:50:54.579 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-26T15:50:54.579 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-26T15:50:54.579 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-26T15:50:54.579 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-26T15:50:54.728 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: Writing manifest to image destination 2024-04-26T15:50:54.840 INFO:teuthology.orchestra.run.smithi059.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-26T15:50:55.205 INFO:teuthology.orchestra.run.smithi059.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-26T15:50:55.739 INFO:teuthology.orchestra.run.smithi059.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-26T15:50:55.739 INFO:teuthology.orchestra.run.smithi059.stderr:Extracting ceph user uid/gid from container image... 2024-04-26T15:50:56.163 INFO:teuthology.orchestra.run.smithi059.stderr:stat: 167 167 2024-04-26T15:50:56.720 INFO:teuthology.orchestra.run.smithi059.stderr:Creating initial keys... 2024-04-26T15:50:57.143 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-authtool: AQBhzStmVEy9AxAAT7p7ZBRJFLcOKIyYWPK+8A== 2024-04-26T15:50:58.132 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-authtool: AQBizStmUYMFAxAAPde9V6WMjKfdaiHCLowEwg== 2024-04-26T15:50:59.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-authtool: AQBizStmJbL+NhAA0Ipfh7XkFBbNJOtOPG/GcQ== 2024-04-26T15:50:59.403 INFO:teuthology.orchestra.run.smithi059.stderr:Creating initial monmap... 2024-04-26T15:50:59.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T15:50:59.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-26T15:50:59.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:50:59.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T15:51:00.324 INFO:teuthology.orchestra.run.smithi059.stderr:monmaptool for a [v2:172.21.15.59:3300,v1:172.21.15.59:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T15:51:00.324 INFO:teuthology.orchestra.run.smithi059.stderr:setting min_mon_release = octopus 2024-04-26T15:51:00.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/monmaptool: set fsid to b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:51:00.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T15:51:00.324 INFO:teuthology.orchestra.run.smithi059.stderr: 2024-04-26T15:51:00.324 INFO:teuthology.orchestra.run.smithi059.stderr:Creating mon... 2024-04-26T15:51:01.179 INFO:teuthology.orchestra.run.smithi059.stderr:create mon.a on 2024-04-26T15:51:01.392 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-26T15:51:01.548 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-b6607c98-03e4-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-b6607c98-03e4-11ef-bc93-c7b262605968.target. 2024-04-26T15:51:01.549 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-b6607c98-03e4-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-b6607c98-03e4-11ef-bc93-c7b262605968.target. 2024-04-26T15:51:01.856 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: Failed to reset failed state of unit ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-26T15:51:01.864 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: Created symlink /etc/systemd/system/ceph-b6607c98-03e4-11ef-bc93-c7b262605968.target.wants/ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-b6607c98-03e4-11ef-bc93-c7b262605968@.service. 2024-04-26T15:51:02.503 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: disabled 2024-04-26T15:51:02.510 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: inactive 2024-04-26T15:51:02.511 INFO:teuthology.orchestra.run.smithi059.stderr:firewalld.service is not enabled 2024-04-26T15:51:02.511 INFO:teuthology.orchestra.run.smithi059.stderr:Not possible to enable service . firewalld.service is not available 2024-04-26T15:51:02.511 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for mon to start... 2024-04-26T15:51:02.511 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for mon... 2024-04-26T15:51:02.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:02 smithi059 systemd[1]: Started Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:51:03.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: cluster: 2024-04-26T15:51:03.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: id: b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:51:03.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-26T15:51:03.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: services: 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.547379s) 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: data: 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: pgs: 2024-04-26T15:51:03.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:03.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:03 smithi059 ceph-mon[21471]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T15:51:03.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:03 smithi059 ceph-mon[21471]: monmap e1: 1 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0]} 2024-04-26T15:51:03.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:03 smithi059 ceph-mon[21471]: fsmap 2024-04-26T15:51:03.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:03 smithi059 ceph-mon[21471]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T15:51:03.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:03 smithi059 ceph-mon[21471]: mgrmap e1: no daemons active 2024-04-26T15:51:03.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:03 smithi059 ceph-mon[21471]: from='client.? 172.21.15.59:0/3243983335' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T15:51:03.541 INFO:teuthology.orchestra.run.smithi059.stderr:mon is available 2024-04-26T15:51:03.541 INFO:teuthology.orchestra.run.smithi059.stderr:Assimilating anything we can from ceph.conf... 2024-04-26T15:51:04.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:04.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: [global] 2024-04-26T15:51:04.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: fsid = b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.59:3300,v1:172.21.15.59:6789] 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: [mgr] 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: [osd] 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-26T15:51:04.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-26T15:51:04.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:04 smithi059 ceph-mon[21471]: from='client.? 172.21.15.59:0/618451553' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T15:51:04.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:04 smithi059 ceph-mon[21471]: from='client.? 172.21.15.59:0/618451553' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-26T15:51:04.597 INFO:teuthology.orchestra.run.smithi059.stderr:Generating new minimal ceph.conf... 2024-04-26T15:51:05.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:05 smithi059 ceph-mon[21471]: from='client.? 172.21.15.59:0/2564220240' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:51:05.587 INFO:teuthology.orchestra.run.smithi059.stderr:Restarting the monitor... 2024-04-26T15:51:05.922 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:05 smithi059 systemd[1]: Stopping Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:51:05.923 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:05 smithi059 bash[21690]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-26T15:51:05.923 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a[21466]: 2024-04-26T15:51:05.737+0000 7f14c006e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T15:51:05.923 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a[21466]: 2024-04-26T15:51:05.737+0000 7f14c006e700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-26T15:51:06.239 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:05 smithi059 podman[21703]: 2024-04-26 15:51:05.922797652 +0000 UTC m=+0.272998777 container died 456d24d3c5bb8a13e103ca416df8e4f47f5bb9ed5f838f5f0bbed30ffc953459 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, version=8, vendor=Red Hat, Inc., name=centos-stream, release=754, io.openshift.tags=base centos centos-stream, ceph=True, distribution-scope=public, GIT_BRANCH=HEAD, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-26T15:51:06.495 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 podman[21703]: 2024-04-26 15:51:06.270910951 +0000 UTC m=+0.621112077 container cleanup 456d24d3c5bb8a13e103ca416df8e4f47f5bb9ed5f838f5f0bbed30ffc953459 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, version=8, ceph=True, com.redhat.component=centos-stream-container, GIT_CLEAN=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, name=centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-26T15:51:06.495 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 bash[21703]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a 2024-04-26T15:51:06.495 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 bash[21730]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-26T15:51:06.772 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 podman[21718]: 2024-04-26 15:51:06.496741114 +0000 UTC m=+0.575376115 container remove 456d24d3c5bb8a13e103ca416df8e4f47f5bb9ed5f838f5f0bbed30ffc953459 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, name=centos-stream, version=8, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.component=centos-stream-container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=) 2024-04-26T15:51:06.772 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T15:51:06.772 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 systemd[1]: Stopped Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:51:06.772 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 systemd[1]: Starting Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:51:06.772 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 podman[21800]: 2024-04-26 15:51:06.647839307 +0000 UTC m=+0.018884259 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-26T15:51:06.772 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 podman[21800]: 2024-04-26 15:51:06.773241104 +0000 UTC m=+0.144286094 container create 052d9cc2e6db782849eb0f4822c9d92e2fc8b7388d29dbed0216ebc8a49998e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, release=754, GIT_CLEAN=True, io.openshift.expose-services=, name=centos-stream, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True) 2024-04-26T15:51:07.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 podman[21800]: 2024-04-26 15:51:06.959524006 +0000 UTC m=+0.330568951 container init 052d9cc2e6db782849eb0f4822c9d92e2fc8b7388d29dbed0216ebc8a49998e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, ceph=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, release=754) 2024-04-26T15:51:07.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 podman[21800]: 2024-04-26 15:51:06.962541638 +0000 UTC m=+0.333586578 container start 052d9cc2e6db782849eb0f4822c9d92e2fc8b7388d29dbed0216ebc8a49998e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.expose-services=, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, release=754, RELEASE=HEAD, GIT_BRANCH=HEAD, version=8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, ceph=True, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-26T15:51:07.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T15:51:07.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-26T15:51:07.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: pidfile_write: ignore empty --pid-file 2024-04-26T15:51:07.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: load: jerasure load: lrc 2024-04-26T15:51:07.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: RocksDB version: 6.15.5 2024-04-26T15:51:07.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Compile date Apr 18 2022 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: DB SUMMARY 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: DB Session ID: ODGML4QFRO9H0D8TD0CS 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: CURRENT file: CURRENT 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82627 ; 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.error_if_exists: 0 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.create_if_missing: 0 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.paranoid_checks: 1 2024-04-26T15:51:07.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.env: 0x55b3290c6860 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.fs: Posix File System 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.info_log: 0x55b32a085dc0 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.statistics: (nil) 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.use_fsync: 0 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_log_file_size: 0 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.allow_fallocate: 1 2024-04-26T15:51:07.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.use_direct_reads: 0 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.db_log_dir: 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T15:51:07.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T15:51:07.046 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T15:51:07.046 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.write_buffer_manager: 0x55b32a1762a0 2024-04-26T15:51:07.046 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T15:51:07.046 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T15:51:07.046 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T15:51:07.046 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T15:51:07.046 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T15:51:07.046 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T15:51:07.046 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.unordered_write: 0 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.row_cache: None 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.wal_filter: None 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.preserve_deletes: 0 2024-04-26T15:51:07.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.two_write_queues: 0 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.atomic_flush: 0 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.log_readahead_size: 0 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_background_jobs: 2 2024-04-26T15:51:07.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_background_compactions: -1 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_subcompactions: 1 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_open_files: -1 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T15:51:07.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_background_flushes: -1 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Compression algorithms supported: 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: kZSTD supported: 0 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: kXpressCompression supported: 0 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: kLZ4Compression supported: 1 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: kBZip2Compression supported: 0 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: kZlibCompression supported: 1 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: kSnappyCompression supported: 1 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T15:51:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.merge_operator: 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_filter: None 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b32a051d00) 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout: cache_index_and_filter_blocks: 1 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout: pin_top_level_index_and_filter: 1 2024-04-26T15:51:07.051 INFO:journalctl@ceph.mon.a.smithi059.stdout: index_type: 0 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: data_block_index_type: 0 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: index_shortening: 1 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: hash_index_allow_collision: 1 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: checksum: 1 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: no_block_cache: 0 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_cache: 0x55b32a0bc170 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_cache_name: BinnedLRUCache 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_cache_options: 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: capacity : 536870912 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: num_shard_bits : 4 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: strict_capacity_limit : 0 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: high_pri_pool_ratio: 0.000 2024-04-26T15:51:07.052 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_cache_compressed: (nil) 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: persistent_cache: (nil) 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_size: 4096 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_size_deviation: 10 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_restart_interval: 16 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: index_block_restart_interval: 1 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: metadata_block_size: 4096 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: partition_filters: 0 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_delta_encoding: 1 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: whole_key_filtering: 1 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: verify_compression: 0 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: read_amp_bytes_per_bit: 0 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: format_version: 4 2024-04-26T15:51:07.053 INFO:journalctl@ceph.mon.a.smithi059.stdout: enable_index_compression: 1 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_align: 0 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compression: NoCompression 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.num_levels: 7 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T15:51:07.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T15:51:07.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T15:51:07.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T15:51:07.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.table_properties_collectors: 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.inplace_update_support: 0 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.bloom_locality: 0 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.max_successive_merges: 0 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T15:51:07.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.ttl: 2592000 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.enable_blob_files: false 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.min_blob_size: 0 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-26T15:51:07.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714146666989802, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-26T15:51:07.060 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-26T15:51:07.060 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T15:51:07.060 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714146666990898, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79177, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77440, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71749, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714146666, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "419b6a46-82f7-42e5-9231-597832c6caef", "db_session_id": "ODGML4QFRO9H0D8TD0CS"}} 2024-04-26T15:51:07.060 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-26T15:51:07.060 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714146666991371, "job": 1, "event": "recovery_finished"} 2024-04-26T15:51:07.060 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T15:51:07.060 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55b32a0a2a80 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: DB pointer 0x55b32a0b2000 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** DB Stats ** 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T15:51:07.061 INFO:journalctl@ceph.mon.a.smithi059.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 106.5 0.00 0.00 1 0.001 0 0 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 106.5 0.00 0.00 1 0.001 0 0 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 106.5 0.00 0.00 1 0.001 0 0 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 106.5 0.00 0.00 1 0.001 0 0 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T15:51:07.062 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: Cumulative compaction: 0.00 GB write, 17.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: Interval compaction: 0.00 GB write, 17.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 106.5 0.00 0.00 1 0.001 0 0 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 106.5 0.00 0.00 1 0.001 0 0 2024-04-26T15:51:07.063 INFO:journalctl@ceph.mon.a.smithi059.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 106.5 0.00 0.00 1 0.001 0 0 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: Cumulative compaction: 0.00 GB write, 17.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T15:51:07.064 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: starting mon.a rank 0 at public addrs [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] at bind addrs [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: mon.a@-1(???) e1 preinit fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: mon.a@-1(???).mds e1 new map 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: mon.a@-1(???).mds e1 print_map 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout: e1 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout: legacy client fscid: -1 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout: No filesystems configured 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T15:51:07.065 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta expand map: {default=false} 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta from 'false' to 'false' 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta expanded map: {default=false} 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta expand map: {default=info} 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta from 'info' to 'info' 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta expanded map: {default=info} 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta expand map: {default=daemon} 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta from 'daemon' to 'daemon' 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta expanded map: {default=daemon} 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta expand map: {default=debug} 2024-04-26T15:51:07.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta from 'debug' to 'debug' 2024-04-26T15:51:07.067 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:06 smithi059 ceph-mon[21819]: expand_channel_meta expanded map: {default=debug} 2024-04-26T15:51:07.067 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:07 smithi059 bash[21800]: 052d9cc2e6db782849eb0f4822c9d92e2fc8b7388d29dbed0216ebc8a49998e1 2024-04-26T15:51:07.067 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:07 smithi059 systemd[1]: Started Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:51:07.067 INFO:teuthology.orchestra.run.smithi059.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-26T15:51:07.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:07 smithi059 ceph-mon[21819]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T15:51:07.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:07 smithi059 ceph-mon[21819]: monmap e1: 1 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0]} 2024-04-26T15:51:07.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:07 smithi059 ceph-mon[21819]: fsmap 2024-04-26T15:51:07.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:07 smithi059 ceph-mon[21819]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T15:51:07.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:07 smithi059 ceph-mon[21819]: mgrmap e1: no daemons active 2024-04-26T15:51:08.175 INFO:teuthology.orchestra.run.smithi059.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-26T15:51:08.175 INFO:teuthology.orchestra.run.smithi059.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-26T15:51:08.175 INFO:teuthology.orchestra.run.smithi059.stderr:Creating mgr... 2024-04-26T15:51:08.175 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying port 9283 ... 2024-04-26T15:51:08.357 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: Failed to reset failed state of unit ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-26T15:51:08.365 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: Created symlink /etc/systemd/system/ceph-b6607c98-03e4-11ef-bc93-c7b262605968.target.wants/ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-b6607c98-03e4-11ef-bc93-c7b262605968@.service. 2024-04-26T15:51:08.789 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:08 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/756579180' entity='client.admin' 2024-04-26T15:51:08.790 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:08 smithi059 podman[22037]: 2024-04-26 15:51:08.656806618 +0000 UTC m=+0.019039718 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-26T15:51:09.050 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:08 smithi059 podman[22037]: 2024-04-26 15:51:08.7905079 +0000 UTC m=+0.152741032 container create 62b620913a03c3654aa6d068692434a6cc9d782ff986351840a534b4d86db799 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, name=centos-stream, ceph=True, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, version=8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git) 2024-04-26T15:51:09.050 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:08 smithi059 podman[22037]: 2024-04-26 15:51:08.967988471 +0000 UTC m=+0.330221558 container init 62b620913a03c3654aa6d068692434a6cc9d782ff986351840a534b4d86db799 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, distribution-scope=public, architecture=x86_64, RELEASE=HEAD, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T15:51:09.050 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:08 smithi059 podman[22037]: 2024-04-26 15:51:08.970544816 +0000 UTC m=+0.332777911 container start 62b620913a03c3654aa6d068692434a6cc9d782ff986351840a534b4d86db799 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, ceph=True, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_CLEAN=True) 2024-04-26T15:51:09.050 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:09 smithi059 bash[22037]: 62b620913a03c3654aa6d068692434a6cc9d782ff986351840a534b4d86db799 2024-04-26T15:51:09.063 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: disabled 2024-04-26T15:51:09.071 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: inactive 2024-04-26T15:51:09.071 INFO:teuthology.orchestra.run.smithi059.stderr:firewalld.service is not enabled 2024-04-26T15:51:09.072 INFO:teuthology.orchestra.run.smithi059.stderr:Not possible to enable service . firewalld.service is not available 2024-04-26T15:51:09.080 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: disabled 2024-04-26T15:51:09.087 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: inactive 2024-04-26T15:51:09.088 INFO:teuthology.orchestra.run.smithi059.stderr:firewalld.service is not enabled 2024-04-26T15:51:09.088 INFO:teuthology.orchestra.run.smithi059.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-26T15:51:09.088 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for mgr to start... 2024-04-26T15:51:09.088 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for mgr... 2024-04-26T15:51:09.322 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:09 smithi059 systemd[1]: Started Ceph mgr.y for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:51:09.322 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:09.119+0000 7f802cfa6000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T15:51:09.322 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:09.257+0000 7f802cfa6000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:51:09.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:09.362+0000 7f802cfa6000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: { 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "fsid": "b6607c98-03e4-11ef-bc93-c7b262605968", 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "health": { 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "checks": {}, 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "mutes": [] 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "quorum": [ 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 0 2024-04-26T15:51:09.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: ], 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "a" 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: ], 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "monmap": { 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "osdmap": { 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-26T15:51:09.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-26T15:51:09.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-26T15:51:09.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-26T15:51:09.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:09.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "pgmap": { 2024-04-26T15:51:09.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-26T15:51:09.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-26T15:51:09.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-26T15:51:09.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "fsmap": { 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "available": false, 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-26T15:51:09.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "modules": [ 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "iostat", 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "nfs", 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "restful" 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: ], 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "servicemap": { 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "modified": "2024-04-26T15:51:02.484853+0000", 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-26T15:51:09.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: } 2024-04-26T15:51:09.947 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:09 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/945863882' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T15:51:09.947 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:09.694+0000 7f802cfa6000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T15:51:09.947 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:09.779+0000 7f802cfa6000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T15:51:10.259 INFO:teuthology.orchestra.run.smithi059.stderr:mgr not available, waiting (1/15)... 2024-04-26T15:51:11.151 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:10 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:10.982+0000 7f802cfa6000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T15:51:11.151 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:11 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:11.151+0000 7f802cfa6000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T15:51:11.558 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:11 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:11.235+0000 7f802cfa6000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T15:51:11.558 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:11 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:11.558+0000 7f802cfa6000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T15:51:11.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:11 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:11.651+0000 7f802cfa6000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T15:51:11.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:11 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:11.746+0000 7f802cfa6000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T15:51:12.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:11 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:11.994+0000 7f802cfa6000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:51:12.650 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:12.458+0000 7f802cfa6000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T15:51:12.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:12.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: { 2024-04-26T15:51:12.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "fsid": "b6607c98-03e4-11ef-bc93-c7b262605968", 2024-04-26T15:51:12.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "health": { 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "checks": {}, 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "mutes": [] 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "quorum": [ 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 0 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: ], 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "a" 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: ], 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "monmap": { 2024-04-26T15:51:12.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:12.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-26T15:51:12.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-26T15:51:12.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:12.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "osdmap": { 2024-04-26T15:51:12.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:12.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-26T15:51:12.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-26T15:51:12.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-26T15:51:12.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-26T15:51:12.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-26T15:51:12.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-26T15:51:12.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:12.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "pgmap": { 2024-04-26T15:51:12.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-26T15:51:12.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-26T15:51:12.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "fsmap": { 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "available": false, 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-26T15:51:12.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "modules": [ 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "iostat", 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "nfs", 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "restful" 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: ], 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "servicemap": { 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "modified": "2024-04-26T15:51:02.484853+0000", 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-26T15:51:12.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: } 2024-04-26T15:51:13.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:12 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/574981591' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T15:51:13.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:12.970+0000 7f802cfa6000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T15:51:13.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:13.067+0000 7f802cfa6000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T15:51:13.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:13.375+0000 7f802cfa6000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T15:51:13.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:13.468+0000 7f802cfa6000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T15:51:14.247 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:13.894+0000 7f802cfa6000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T15:51:14.247 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:14.035+0000 7f802cfa6000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T15:51:14.247 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:14.130+0000 7f802cfa6000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T15:51:14.247 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:14.248+0000 7f802cfa6000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T15:51:14.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:14.339+0000 7f802cfa6000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T15:51:15.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:14.910+0000 7f802cfa6000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T15:51:15.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:15.282+0000 7f802cfa6000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T15:51:15.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: Activating manager daemon y 2024-04-26T15:51:15.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: mgrmap e2: y(active, starting, since 0.00275232s) 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: from='mgr.14100 172.21.15.59:0/12384484' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: from='mgr.14100 172.21.15.59:0/12384484' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: from='mgr.14100 172.21.15.59:0/12384484' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: from='mgr.14100 172.21.15.59:0/12384484' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: from='mgr.14100 172.21.15.59:0/12384484' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: Manager daemon y is now available 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: from='mgr.14100 172.21.15.59:0/12384484' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: from='mgr.14100 172.21.15.59:0/12384484' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: from='mgr.14100 172.21.15.59:0/12384484' entity='mgr.y' 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: from='mgr.14100 172.21.15.59:0/12384484' entity='mgr.y' 2024-04-26T15:51:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:15 smithi059 ceph-mon[21819]: from='mgr.14100 172.21.15.59:0/12384484' entity='mgr.y' 2024-04-26T15:51:17.421 INFO:teuthology.orchestra.run.smithi059.stderr:mgr not available, waiting (2/15)... 2024-04-26T15:51:17.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:17 smithi059 ceph-mon[21819]: mgrmap e3: y(active, since 1.00678s) 2024-04-26T15:51:19.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:19 smithi059 ceph-mon[21819]: mgrmap e4: y(active, since 2s) 2024-04-26T15:51:20.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:20.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: { 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "fsid": "b6607c98-03e4-11ef-bc93-c7b262605968", 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "health": { 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "checks": {}, 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "mutes": [] 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "quorum": [ 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 0 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: ], 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "a" 2024-04-26T15:51:20.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: ], 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "quorum_age": 13, 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "monmap": { 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "osdmap": { 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-26T15:51:20.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-26T15:51:20.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-26T15:51:20.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-26T15:51:20.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:20.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "pgmap": { 2024-04-26T15:51:20.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-26T15:51:20.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-26T15:51:20.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-26T15:51:20.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "fsmap": { 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "available": true, 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "modules": [ 2024-04-26T15:51:20.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "iostat", 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "nfs", 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "restful" 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: ], 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "servicemap": { 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "modified": "2024-04-26T15:51:02.484853+0000", 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "services": {} 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: }, 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-26T15:51:20.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: } 2024-04-26T15:51:20.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:20 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/23311012' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T15:51:20.695 INFO:teuthology.orchestra.run.smithi059.stderr:mgr is available 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: [global] 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: fsid = b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: [mgr] 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 2024-04-26T15:51:21.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: [osd] 2024-04-26T15:51:21.290 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-26T15:51:21.290 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-26T15:51:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:21 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1188359772' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T15:51:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:21 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1188359772' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-26T15:51:21.701 INFO:teuthology.orchestra.run.smithi059.stderr:Enabling cephadm module... 2024-04-26T15:51:23.306 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:23 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2355786782' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-26T15:51:23.306 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ignoring --setuser ceph since I am not root 2024-04-26T15:51:23.306 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ignoring --setgroup ceph since I am not root 2024-04-26T15:51:23.306 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:23.172+0000 7f61059be000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T15:51:23.307 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:23.306+0000 7f61059be000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:51:23.579 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:23.407+0000 7f61059be000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T15:51:23.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:23.730+0000 7f61059be000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T15:51:23.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:23.815+0000 7f61059be000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T15:51:24.329 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:24 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2355786782' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-26T15:51:24.329 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:24 smithi059 ceph-mon[21819]: mgrmap e5: y(active, since 7s) 2024-04-26T15:51:24.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: { 2024-04-26T15:51:24.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-26T15:51:24.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "available": true, 2024-04-26T15:51:24.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-26T15:51:24.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-26T15:51:24.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: } 2024-04-26T15:51:25.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:25 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2056403491' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T15:51:25.256 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:25.002+0000 7f61059be000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T15:51:25.257 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:25.172+0000 7f61059be000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T15:51:25.257 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:25.256+0000 7f61059be000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T15:51:26.019 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:25.581+0000 7f61059be000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T15:51:26.019 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:25.674+0000 7f61059be000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T15:51:26.019 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:25.770+0000 7f61059be000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T15:51:26.019 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:26 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:26.019+0000 7f61059be000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:51:26.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:26 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:26.479+0000 7f61059be000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T15:51:27.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:26 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:26.982+0000 7f61059be000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T15:51:27.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:27.079+0000 7f61059be000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T15:51:27.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:27.386+0000 7f61059be000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T15:51:27.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:27.481+0000 7f61059be000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T15:51:28.262 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:27.907+0000 7f61059be000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T15:51:28.262 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:28.048+0000 7f61059be000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T15:51:28.263 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:28.143+0000 7f61059be000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T15:51:28.263 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:28.262+0000 7f61059be000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T15:51:28.514 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:28.356+0000 7f61059be000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T15:51:28.514 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for the mgr to restart... 2024-04-26T15:51:28.514 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for mgr epoch 5... 2024-04-26T15:51:29.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:28.935+0000 7f61059be000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T15:51:29.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:29 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:29.305+0000 7f61059be000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T15:51:30.368 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: { 2024-04-26T15:51:30.368 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-26T15:51:30.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "initialized": true 2024-04-26T15:51:30.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: } 2024-04-26T15:51:30.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:51:30] ENGINE Bus STARTING 2024-04-26T15:51:30.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:51:30] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T15:51:30.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:51:30] ENGINE Bus STARTED 2024-04-26T15:51:30.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: Active manager daemon y restarted 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: Activating manager daemon y 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: osdmap e2: 0 total, 0 up, 0 in 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: mgrmap e6: y(active, starting, since 0.0524801s) 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: Manager daemon y is now available 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:51:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:51:30.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:30 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:51:30.979 INFO:teuthology.orchestra.run.smithi059.stderr:mgr epoch 5 is available 2024-04-26T15:51:30.979 INFO:teuthology.orchestra.run.smithi059.stderr:Setting orchestrator backend to cephadm... 2024-04-26T15:51:31.512 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:31 smithi059 ceph-mon[21819]: mgrmap e7: y(active, since 1.055s) 2024-04-26T15:51:31.512 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:31 smithi059 ceph-mon[21819]: [26/Apr/2024:15:51:30] ENGINE Bus STARTING 2024-04-26T15:51:31.512 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:31 smithi059 ceph-mon[21819]: [26/Apr/2024:15:51:30] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T15:51:31.512 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:31 smithi059 ceph-mon[21819]: [26/Apr/2024:15:51:30] ENGINE Bus STARTED 2024-04-26T15:51:31.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:31 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:31.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:31 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:51:32.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:32 smithi059 ceph-mon[21819]: mgrmap e8: y(active, since 2s) 2024-04-26T15:51:32.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:32 smithi059 ceph-mon[21819]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:51:32.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:32 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:32.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:32 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:51:33.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: value unchanged 2024-04-26T15:51:33.622 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:33 smithi059 ceph-mon[21819]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:51:33.624 INFO:teuthology.orchestra.run.smithi059.stderr:Generating ssh key... 2024-04-26T15:51:34.761 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: Generating public/private rsa key pair. 2024-04-26T15:51:34.761 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: Your identification has been saved in /tmp/tmp0tr2jz2r/key. 2024-04-26T15:51:34.761 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: Your public key has been saved in /tmp/tmp0tr2jz2r/key.pub. 2024-04-26T15:51:34.761 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: The key fingerprint is: 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: SHA256:l22iVEYrQqR1R5b/7okDbsgllwMc41MvM5mZLkq7eeI ceph-b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: The key's randomart image is: 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: +---[RSA 3072]----+ 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: | .+ ..=. | 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: | + .o+o. | 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: | . .o.++B | 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: | .=+Ooo | 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: | S++=o. | 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: | ..ooBo . | 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: | . +.B o . | 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: | +.+ o .... | 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: | .E+ . .oo | 2024-04-26T15:51:34.762 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: +----[SHA256]-----+ 2024-04-26T15:51:35.337 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDyM8KwyRvZ2qpIs82Wh+P6xmIVcj8NJdKqSpe4GFgP3+KX/1y+C23iIT3jrwvuhoVgEXf+Y3xDPujKLkFF18sYBkkf7z6XaGlW2LDPqNymPhh1ut/R+i3FfgA1bAeQkAZ+dkxBXAL0LMT2zBqjWd4TSTIg47btmfHQC7DLfauUvJBKSTArw6MGhNJ9cXj+STKkh5gIvyX6pi3E0umBtxQX4xLlsUCD+29iH+J0ThrN7RicIYGxzBCOqitlZKiDriPF6QZcrg3t9dNZ95aCKbBtgDljiLRdBaRAt5uvX6tG4FKHwxobAs0VtRYLnPj9lgZtaYT5PijDZ0oz9tKXnV2lajGDhuLzuIZZHGUL5/uULviUxAAdA51zDvoDd9R3giVdmHsGOH/1nzzAVXPwJ2/EmgUY3TAJC6QV87wdLHF/Ayt6bVCqPUmyOEOLHzrM3+NYXloycyInb1KsVkjdEHMO8WQ0t4Iz5NZomtCOzcdoAqkx0d/7NZrLRA+U0Nr7q0= ceph-b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:51:35.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:35 smithi059 ceph-mon[21819]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:51:35.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:35 smithi059 ceph-mon[21819]: Generating ssh key... 2024-04-26T15:51:35.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:35 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:35.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:35 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:35.912 INFO:teuthology.orchestra.run.smithi059.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-26T15:51:35.912 INFO:teuthology.orchestra.run.smithi059.stderr:Adding key to root@localhost authorized_keys... 2024-04-26T15:51:35.913 INFO:teuthology.orchestra.run.smithi059.stderr:Adding host smithi059... 2024-04-26T15:51:36.497 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:36 smithi059 ceph-mon[21819]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:51:37.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: Added host 'smithi059' with addr '172.21.15.59' 2024-04-26T15:51:37.943 INFO:teuthology.orchestra.run.smithi059.stderr:Deploying unmanaged mon service... 2024-04-26T15:51:38.196 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:38 smithi059 ceph-mon[21819]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi059", "addr": "172.21.15.59", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:51:38.197 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:38 smithi059 ceph-mon[21819]: Deploying cephadm binary to smithi059 2024-04-26T15:51:38.197 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:38 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:38.197 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:38 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:51:38.609 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-26T15:51:39.243 INFO:teuthology.orchestra.run.smithi059.stderr:Deploying unmanaged mgr service... 2024-04-26T15:51:39.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:39 smithi059 ceph-mon[21819]: Added host smithi059 2024-04-26T15:51:39.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:39 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:40.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-26T15:51:40.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:40 smithi059 ceph-mon[21819]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:51:40.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:40 smithi059 ceph-mon[21819]: Saving service mon spec with placement count:5 2024-04-26T15:51:41.286 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:41 smithi059 ceph-mon[21819]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:51:41.286 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:41 smithi059 ceph-mon[21819]: Saving service mgr spec with placement count:2 2024-04-26T15:51:41.286 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:41 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:41.286 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:41 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:42.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:42 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:42.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:42 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3909870060' entity='client.admin' 2024-04-26T15:51:44.092 INFO:teuthology.orchestra.run.smithi059.stderr:Enabling the dashboard module... 2024-04-26T15:51:44.660 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:44 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:44.660 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:44 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2752555530' entity='client.admin' 2024-04-26T15:51:44.660 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:44 smithi059 ceph-mon[21819]: from='mgr.14118 172.21.15.59:0/3211918258' entity='mgr.y' 2024-04-26T15:51:45.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:45 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/142332300' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-26T15:51:45.511 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ignoring --setuser ceph since I am not root 2024-04-26T15:51:45.511 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ignoring --setgroup ceph since I am not root 2024-04-26T15:51:45.793 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:45.559+0000 7f86a2651000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T15:51:45.793 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:45.693+0000 7f86a2651000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:51:46.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:45.796+0000 7f86a2651000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T15:51:46.389 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:46.127+0000 7f86a2651000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T15:51:46.389 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:46.213+0000 7f86a2651000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T15:51:46.814 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:46 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/142332300' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-26T15:51:46.814 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:46 smithi059 ceph-mon[21819]: mgrmap e9: y(active, since 16s) 2024-04-26T15:51:46.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: { 2024-04-26T15:51:46.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-26T15:51:46.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "available": true, 2024-04-26T15:51:46.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-26T15:51:46.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-26T15:51:46.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: } 2024-04-26T15:51:47.470 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for the mgr to restart... 2024-04-26T15:51:47.470 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for mgr epoch 9... 2024-04-26T15:51:47.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:47 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1297073591' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T15:51:47.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:47.419+0000 7f86a2651000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T15:51:47.852 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:47.595+0000 7f86a2651000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T15:51:47.853 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:47.682+0000 7f86a2651000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T15:51:48.107 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:48.013+0000 7f86a2651000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T15:51:48.107 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:48.107+0000 7f86a2651000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T15:51:48.453 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:48.204+0000 7f86a2651000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T15:51:48.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:48.453+0000 7f86a2651000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:51:49.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:48.918+0000 7f86a2651000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T15:51:49.825 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:49.421+0000 7f86a2651000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T15:51:49.826 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:49.519+0000 7f86a2651000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T15:51:49.826 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:49.825+0000 7f86a2651000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T15:51:50.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:49.918+0000 7f86a2651000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T15:51:50.688 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:50.336+0000 7f86a2651000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T15:51:50.689 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:50.475+0000 7f86a2651000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T15:51:50.689 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:50.571+0000 7f86a2651000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T15:51:50.689 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:50.689+0000 7f86a2651000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T15:51:51.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:50.782+0000 7f86a2651000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T15:51:51.726 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:51 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:51.355+0000 7f86a2651000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T15:51:51.984 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:51 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:51:51.727+0000 7f86a2651000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T15:51:51.984 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:51 smithi059 ceph-mon[21819]: Active manager daemon y restarted 2024-04-26T15:51:51.985 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:51 smithi059 ceph-mon[21819]: Activating manager daemon y 2024-04-26T15:51:51.985 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:51 smithi059 ceph-mon[21819]: osdmap e3: 0 total, 0 up, 0 in 2024-04-26T15:51:52.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: { 2024-04-26T15:51:52.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-26T15:51:52.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: "initialized": true 2024-04-26T15:51:52.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: } 2024-04-26T15:51:53.080 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: mgrmap e10: y(active, starting, since 0.0530297s) 2024-04-26T15:51:53.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:51:53.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T15:51:53.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:51:53.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:51:53.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:51:53.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: Manager daemon y is now available 2024-04-26T15:51:53.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:51:53.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:51:53.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:51:53.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:51:53.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:51:53] ENGINE Bus STARTING 2024-04-26T15:51:53.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:51:53] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T15:51:53.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:51:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:51:53] ENGINE Bus STARTED 2024-04-26T15:51:53.571 INFO:teuthology.orchestra.run.smithi059.stderr:mgr epoch 9 is available 2024-04-26T15:51:53.571 INFO:teuthology.orchestra.run.smithi059.stderr:Generating a dashboard self-signed certificate... 2024-04-26T15:51:53.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:53 smithi059 ceph-mon[21819]: mgrmap e11: y(active, since 1.06095s) 2024-04-26T15:51:53.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:53 smithi059 ceph-mon[21819]: [26/Apr/2024:15:51:53] ENGINE Bus STARTING 2024-04-26T15:51:53.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:53 smithi059 ceph-mon[21819]: [26/Apr/2024:15:51:53] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T15:51:53.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:53 smithi059 ceph-mon[21819]: [26/Apr/2024:15:51:53] ENGINE Bus STARTED 2024-04-26T15:51:53.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:53 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:51:54.304 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-26T15:51:54.861 INFO:teuthology.orchestra.run.smithi059.stderr:Creating initial admin user... 2024-04-26T15:51:55.245 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:55 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:51:55.245 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:55 smithi059 ceph-mon[21819]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:51:55.245 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:55 smithi059 ceph-mon[21819]: mgrmap e12: y(active, since 2s) 2024-04-26T15:51:55.245 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:55 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:51:55.245 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:55 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:51:55.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$vyws9x1cVw1CEVlYCyD44.x4pSD5ylPq92m/Rzuw42MiCbF/Hgf.m", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714146715, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-26T15:51:56.327 INFO:teuthology.orchestra.run.smithi059.stderr:Fetching dashboard port number... 2024-04-26T15:51:56.574 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:56 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:51:56.575 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:56 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:51:56.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: 8443 2024-04-26T15:51:57.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:57 smithi059 ceph-mon[21819]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:51:57.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:57 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/586144242' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-26T15:51:58.259 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: disabled 2024-04-26T15:51:58.267 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: inactive 2024-04-26T15:51:58.268 INFO:teuthology.orchestra.run.smithi059.stderr:firewalld.service is not enabled 2024-04-26T15:51:58.268 INFO:teuthology.orchestra.run.smithi059.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-26T15:51:58.270 INFO:teuthology.orchestra.run.smithi059.stderr:Ceph Dashboard is now available at: 2024-04-26T15:51:58.270 INFO:teuthology.orchestra.run.smithi059.stderr: 2024-04-26T15:51:58.270 INFO:teuthology.orchestra.run.smithi059.stderr: URL: https://smithi059:8443/ 2024-04-26T15:51:58.270 INFO:teuthology.orchestra.run.smithi059.stderr: User: admin 2024-04-26T15:51:58.271 INFO:teuthology.orchestra.run.smithi059.stderr: Password: vr550mq346 2024-04-26T15:51:58.271 INFO:teuthology.orchestra.run.smithi059.stderr: 2024-04-26T15:51:58.271 INFO:teuthology.orchestra.run.smithi059.stderr:Enabling autotune for osd_memory_target 2024-04-26T15:51:59.618 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:51:59 smithi059 ceph-mon[21819]: mgrmap e13: y(active, since 6s) 2024-04-26T15:52:00.345 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-26T15:52:01.029 INFO:teuthology.orchestra.run.smithi059.stderr:You can access the Ceph CLI with: 2024-04-26T15:52:01.029 INFO:teuthology.orchestra.run.smithi059.stderr: 2024-04-26T15:52:01.029 INFO:teuthology.orchestra.run.smithi059.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-26T15:52:01.029 INFO:teuthology.orchestra.run.smithi059.stderr: 2024-04-26T15:52:01.030 INFO:teuthology.orchestra.run.smithi059.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-26T15:52:01.030 INFO:teuthology.orchestra.run.smithi059.stderr: 2024-04-26T15:52:01.030 INFO:teuthology.orchestra.run.smithi059.stderr: ceph telemetry on 2024-04-26T15:52:01.030 INFO:teuthology.orchestra.run.smithi059.stderr: 2024-04-26T15:52:01.030 INFO:teuthology.orchestra.run.smithi059.stderr:For more information see: 2024-04-26T15:52:01.030 INFO:teuthology.orchestra.run.smithi059.stderr: 2024-04-26T15:52:01.030 INFO:teuthology.orchestra.run.smithi059.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-26T15:52:01.030 INFO:teuthology.orchestra.run.smithi059.stderr: 2024-04-26T15:52:01.030 INFO:teuthology.orchestra.run.smithi059.stderr:Bootstrap complete. 2024-04-26T15:52:01.072 INFO:tasks.cephadm:Fetching config... 2024-04-26T15:52:01.073 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-26T15:52:01.073 DEBUG:teuthology.orchestra.run.smithi059:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-26T15:52:01.091 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-26T15:52:01.091 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-26T15:52:01.091 DEBUG:teuthology.orchestra.run.smithi059:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-26T15:52:01.148 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-26T15:52:01.148 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-26T15:52:01.148 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd if=/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-26T15:52:01.216 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-26T15:52:01.217 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-26T15:52:01.217 DEBUG:teuthology.orchestra.run.smithi059:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-26T15:52:01.272 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-26T15:52:01.273 DEBUG:teuthology.orchestra.run.smithi059:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDyM8KwyRvZ2qpIs82Wh+P6xmIVcj8NJdKqSpe4GFgP3+KX/1y+C23iIT3jrwvuhoVgEXf+Y3xDPujKLkFF18sYBkkf7z6XaGlW2LDPqNymPhh1ut/R+i3FfgA1bAeQkAZ+dkxBXAL0LMT2zBqjWd4TSTIg47btmfHQC7DLfauUvJBKSTArw6MGhNJ9cXj+STKkh5gIvyX6pi3E0umBtxQX4xLlsUCD+29iH+J0ThrN7RicIYGxzBCOqitlZKiDriPF6QZcrg3t9dNZ95aCKbBtgDljiLRdBaRAt5uvX6tG4FKHwxobAs0VtRYLnPj9lgZtaYT5PijDZ0oz9tKXnV2lajGDhuLzuIZZHGUL5/uULviUxAAdA51zDvoDd9R3giVdmHsGOH/1nzzAVXPwJ2/EmgUY3TAJC6QV87wdLHF/Ayt6bVCqPUmyOEOLHzrM3+NYXloycyInb1KsVkjdEHMO8WQ0t4Iz5NZomtCOzcdoAqkx0d/7NZrLRA+U0Nr7q0= ceph-b6607c98-03e4-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T15:52:01.353 INFO:teuthology.orchestra.run.smithi059.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDyM8KwyRvZ2qpIs82Wh+P6xmIVcj8NJdKqSpe4GFgP3+KX/1y+C23iIT3jrwvuhoVgEXf+Y3xDPujKLkFF18sYBkkf7z6XaGlW2LDPqNymPhh1ut/R+i3FfgA1bAeQkAZ+dkxBXAL0LMT2zBqjWd4TSTIg47btmfHQC7DLfauUvJBKSTArw6MGhNJ9cXj+STKkh5gIvyX6pi3E0umBtxQX4xLlsUCD+29iH+J0ThrN7RicIYGxzBCOqitlZKiDriPF6QZcrg3t9dNZ95aCKbBtgDljiLRdBaRAt5uvX6tG4FKHwxobAs0VtRYLnPj9lgZtaYT5PijDZ0oz9tKXnV2lajGDhuLzuIZZHGUL5/uULviUxAAdA51zDvoDd9R3giVdmHsGOH/1nzzAVXPwJ2/EmgUY3TAJC6QV87wdLHF/Ayt6bVCqPUmyOEOLHzrM3+NYXloycyInb1KsVkjdEHMO8WQ0t4Iz5NZomtCOzcdoAqkx0d/7NZrLRA+U0Nr7q0= ceph-b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:52:01.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDyM8KwyRvZ2qpIs82Wh+P6xmIVcj8NJdKqSpe4GFgP3+KX/1y+C23iIT3jrwvuhoVgEXf+Y3xDPujKLkFF18sYBkkf7z6XaGlW2LDPqNymPhh1ut/R+i3FfgA1bAeQkAZ+dkxBXAL0LMT2zBqjWd4TSTIg47btmfHQC7DLfauUvJBKSTArw6MGhNJ9cXj+STKkh5gIvyX6pi3E0umBtxQX4xLlsUCD+29iH+J0ThrN7RicIYGxzBCOqitlZKiDriPF6QZcrg3t9dNZ95aCKbBtgDljiLRdBaRAt5uvX6tG4FKHwxobAs0VtRYLnPj9lgZtaYT5PijDZ0oz9tKXnV2lajGDhuLzuIZZHGUL5/uULviUxAAdA51zDvoDd9R3giVdmHsGOH/1nzzAVXPwJ2/EmgUY3TAJC6QV87wdLHF/Ayt6bVCqPUmyOEOLHzrM3+NYXloycyInb1KsVkjdEHMO8WQ0t4Iz5NZomtCOzcdoAqkx0d/7NZrLRA+U0Nr7q0= ceph-b6607c98-03e4-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T15:52:01.409 INFO:teuthology.orchestra.run.smithi064.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDyM8KwyRvZ2qpIs82Wh+P6xmIVcj8NJdKqSpe4GFgP3+KX/1y+C23iIT3jrwvuhoVgEXf+Y3xDPujKLkFF18sYBkkf7z6XaGlW2LDPqNymPhh1ut/R+i3FfgA1bAeQkAZ+dkxBXAL0LMT2zBqjWd4TSTIg47btmfHQC7DLfauUvJBKSTArw6MGhNJ9cXj+STKkh5gIvyX6pi3E0umBtxQX4xLlsUCD+29iH+J0ThrN7RicIYGxzBCOqitlZKiDriPF6QZcrg3t9dNZ95aCKbBtgDljiLRdBaRAt5uvX6tG4FKHwxobAs0VtRYLnPj9lgZtaYT5PijDZ0oz9tKXnV2lajGDhuLzuIZZHGUL5/uULviUxAAdA51zDvoDd9R3giVdmHsGOH/1nzzAVXPwJ2/EmgUY3TAJC6QV87wdLHF/Ayt6bVCqPUmyOEOLHzrM3+NYXloycyInb1KsVkjdEHMO8WQ0t4Iz5NZomtCOzcdoAqkx0d/7NZrLRA+U0Nr7q0= ceph-b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:52:01.419 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-26T15:52:01.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:01 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2698584478' entity='client.admin' 2024-04-26T15:52:05.352 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-26T15:52:05.352 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-26T15:52:05.984 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:05 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3978567648' entity='client.admin' 2024-04-26T15:52:05.984 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:05 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:05.984 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:05 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:52:05.984 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:05 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:05.985 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:05 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:05.985 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:05 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:08.038 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi064 2024-04-26T15:52:08.039 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-04-26T15:52:08.039 DEBUG:teuthology.orchestra.run.smithi064:> dd of=/etc/ceph/ceph.conf 2024-04-26T15:52:08.059 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-04-26T15:52:08.059 DEBUG:teuthology.orchestra.run.smithi064:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:52:08.117 INFO:tasks.cephadm:Adding host smithi064 to orchestrator... 2024-04-26T15:52:08.117 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch host add smithi064 2024-04-26T15:52:08.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:08 smithi059 ceph-mon[21819]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:08.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:08 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:08.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:08 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:08.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:08 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:08.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:08 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:08.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:08 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T15:52:08.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:08 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:08.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:08 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:09.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:09 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:52:10.775 INFO:teuthology.orchestra.run.smithi059.stdout:Added host 'smithi064' with addr '172.21.15.64' 2024-04-26T15:52:10.775 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:10 smithi059 ceph-mon[21819]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi064", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:10.775 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:10 smithi059 ceph-mon[21819]: Deploying cephadm binary to smithi064 2024-04-26T15:52:11.358 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-26T15:52:11.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:11.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:11 smithi059 ceph-mon[21819]: Added host smithi064 2024-04-26T15:52:11.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:11.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:11.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:11.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:11.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:13.176 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:52:13.177 INFO:teuthology.orchestra.run.smithi059.stdout:[{"addr": "172.21.15.59", "hostname": "smithi059", "labels": [], "status": ""}, {"addr": "172.21.15.64", "hostname": "smithi064", "labels": [], "status": ""}] 2024-04-26T15:52:13.479 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:13 smithi059 ceph-mon[21819]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:13.479 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:13 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:13.758 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-26T15:52:13.758 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-26T15:52:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:14 smithi059 ceph-mon[21819]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:52:15.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:15 smithi059 ceph-mon[21819]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi064", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:52:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:16.387 INFO:teuthology.orchestra.run.smithi059.stderr:adjusted tunables profile to default 2024-04-26T15:52:16.708 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:16 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/761754527' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-26T15:52:17.005 INFO:tasks.cephadm:Adding mon.a on smithi059 2024-04-26T15:52:17.005 INFO:tasks.cephadm:Adding mon.c on smithi059 2024-04-26T15:52:17.005 INFO:tasks.cephadm:Adding mon.b on smithi064 2024-04-26T15:52:17.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi059:172.21.15.59=a;smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c;smithi064:172.21.15.64=b' 2024-04-26T15:52:17.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:17 smithi059 ceph-mon[21819]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:17.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:17 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/761754527' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T15:52:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:17 smithi059 ceph-mon[21819]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:52:18.827 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled mon update... 2024-04-26T15:52:19.475 DEBUG:teuthology.orchestra.run.smithi059:mon.c> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.c.service 2024-04-26T15:52:19.477 DEBUG:teuthology.orchestra.run.smithi064:mon.b> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.b.service 2024-04-26T15:52:19.479 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T15:52:19.480 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:52:19.497 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:19 smithi059 ceph-mon[21819]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:19.497 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:19 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:19.497 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:19 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:19.497 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:19 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:19.498 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:19 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:19.498 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:19 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:19.498 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:19 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:19.498 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:19 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:19.498 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:19 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:20.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:20 smithi059 ceph-mon[21819]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi059:172.21.15.59=a;smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c;smithi064:172.21.15.64=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:20.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:20 smithi059 ceph-mon[21819]: Saving service mon spec with placement smithi059:172.21.15.59=a;smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c;smithi064:172.21.15.64=b;count:3 2024-04-26T15:52:20.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:20 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T15:52:20.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:20 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:52:20.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:20 smithi059 ceph-mon[21819]: Deploying daemon mon.c on smithi059 2024-04-26T15:52:21.194 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-04-26T15:52:21.195 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":1,"fsid":"b6607c98-03e4-11ef-bc93-c7b262605968","modified":"2024-04-26T15:50:59.741700Z","created":"2024-04-26T15:50:59.741700Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3300","nonce":0},{"type":"v1","addr":"172.21.15.59:6789","nonce":0}]},"addr":"172.21.15.59:6789/0","public_addr":"172.21.15.59:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T15:52:21.196 INFO:teuthology.orchestra.run.smithi064.stderr:dumped monmap epoch 1 2024-04-26T15:52:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:21 smithi059 ceph-mon[21819]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:21 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/1109922982' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:21.975 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:21 smithi059 systemd[1]: Starting Ceph mon.c for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:52:22.229 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:21 smithi059 podman[26518]: 2024-04-26 15:52:21.975800228 +0000 UTC m=+0.104767387 container create 17313040ebe2f77c28d4531a3a9e89a1fa9acef98a9f00e4424d1ceba67fe2a4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, version=8, vcs-type=git, release=754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, distribution-scope=public, maintainer=Guillaume Abrioux ) 2024-04-26T15:52:22.229 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:21 smithi059 podman[26518]: 2024-04-26 15:52:21.890063345 +0000 UTC m=+0.019030514 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-26T15:52:22.229 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 podman[26518]: 2024-04-26 15:52:22.206113334 +0000 UTC m=+0.335080494 container init 17313040ebe2f77c28d4531a3a9e89a1fa9acef98a9f00e4424d1ceba67fe2a4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, ceph=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_CLEAN=True, distribution-scope=public) 2024-04-26T15:52:22.229 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 podman[26518]: 2024-04-26 15:52:22.209115729 +0000 UTC m=+0.338082889 container start 17313040ebe2f77c28d4531a3a9e89a1fa9acef98a9f00e4424d1ceba67fe2a4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., vcs-type=git, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.openshift.expose-services=, version=8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, release=754, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-26T15:52:22.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T15:52:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-26T15:52:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: pidfile_write: ignore empty --pid-file 2024-04-26T15:52:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: load: jerasure load: lrc 2024-04-26T15:52:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: RocksDB version: 6.15.5 2024-04-26T15:52:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T15:52:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Compile date Apr 18 2022 2024-04-26T15:52:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: DB SUMMARY 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: DB Session ID: 78U20HKLJ9KNJTIUL0GL 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: CURRENT file: CURRENT 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.error_if_exists: 0 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.create_if_missing: 0 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.paranoid_checks: 1 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.env: 0x55aa3a0c6860 2024-04-26T15:52:22.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.fs: Posix File System 2024-04-26T15:52:22.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.info_log: 0x55aa3bbfddc0 2024-04-26T15:52:22.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T15:52:22.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.statistics: (nil) 2024-04-26T15:52:22.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.use_fsync: 0 2024-04-26T15:52:22.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_log_file_size: 0 2024-04-26T15:52:22.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T15:52:22.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T15:52:22.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.allow_fallocate: 1 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.use_direct_reads: 0 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.db_log_dir: 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T15:52:22.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.write_buffer_manager: 0x55aa3bcee2d0 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T15:52:22.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T15:52:22.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T15:52:22.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T15:52:22.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T15:52:22.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T15:52:22.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.unordered_write: 0 2024-04-26T15:52:22.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T15:52:22.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T15:52:22.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T15:52:22.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T15:52:22.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.row_cache: None 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.wal_filter: None 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.preserve_deletes: 0 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.two_write_queues: 0 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.atomic_flush: 0 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.log_readahead_size: 0 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T15:52:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_background_jobs: 2 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_background_compactions: -1 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_subcompactions: 1 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_open_files: -1 2024-04-26T15:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_background_flushes: -1 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Compression algorithms supported: 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: kZSTD supported: 0 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: kXpressCompression supported: 0 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: kLZ4Compression supported: 1 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: kBZip2Compression supported: 0 2024-04-26T15:52:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: kZlibCompression supported: 1 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: kSnappyCompression supported: 1 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.merge_operator: 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_filter: None 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55aa3bbc9d00) 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: cache_index_and_filter_blocks: 1 2024-04-26T15:52:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: pin_top_level_index_and_filter: 1 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: index_type: 0 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: data_block_index_type: 0 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: index_shortening: 1 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: hash_index_allow_collision: 1 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: checksum: 1 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: no_block_cache: 0 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_cache: 0x55aa3bc34170 2024-04-26T15:52:22.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_cache_name: BinnedLRUCache 2024-04-26T15:52:22.595 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_cache_options: 2024-04-26T15:52:22.595 INFO:journalctl@ceph.mon.c.smithi059.stdout: capacity : 536870912 2024-04-26T15:52:22.595 INFO:journalctl@ceph.mon.c.smithi059.stdout: num_shard_bits : 4 2024-04-26T15:52:22.595 INFO:journalctl@ceph.mon.c.smithi059.stdout: strict_capacity_limit : 0 2024-04-26T15:52:22.595 INFO:journalctl@ceph.mon.c.smithi059.stdout: high_pri_pool_ratio: 0.000 2024-04-26T15:52:22.595 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_cache_compressed: (nil) 2024-04-26T15:52:22.595 INFO:journalctl@ceph.mon.c.smithi059.stdout: persistent_cache: (nil) 2024-04-26T15:52:22.595 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_size: 4096 2024-04-26T15:52:22.595 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_size_deviation: 10 2024-04-26T15:52:22.595 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_restart_interval: 16 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: index_block_restart_interval: 1 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: metadata_block_size: 4096 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: partition_filters: 0 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_delta_encoding: 1 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: whole_key_filtering: 1 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: verify_compression: 0 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: read_amp_bytes_per_bit: 0 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: format_version: 4 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: enable_index_compression: 1 2024-04-26T15:52:22.596 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_align: 0 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compression: NoCompression 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.num_levels: 7 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T15:52:22.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T15:52:22.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T15:52:22.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T15:52:22.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.table_properties_collectors: 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.inplace_update_support: 0 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.bloom_locality: 0 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.max_successive_merges: 0 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T15:52:22.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.ttl: 2592000 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.enable_blob_files: false 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.min_blob_size: 0 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T15:52:22.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-26T15:52:22.603 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-26T15:52:22.603 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714146742235998, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-26T15:52:22.603 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-26T15:52:22.603 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T15:52:22.603 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714146742236324, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714146742, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "a07ca0c8-7c12-4610-bb5f-baa8a60b3cc4", "db_session_id": "78U20HKLJ9KNJTIUL0GL"}} 2024-04-26T15:52:22.603 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-26T15:52:22.603 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714146742236649, "job": 1, "event": "recovery_finished"} 2024-04-26T15:52:22.603 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T15:52:22.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55aa3bc1aa80 2024-04-26T15:52:22.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: DB pointer 0x55aa3bc2a000 2024-04-26T15:52:22.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-26T15:52:22.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T15:52:22.604 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** DB Stats ** 2024-04-26T15:52:22.604 INFO:journalctl@ceph.mon.c.smithi059.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:52:22.605 INFO:journalctl@ceph.mon.c.smithi059.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T15:52:22.605 INFO:journalctl@ceph.mon.c.smithi059.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T15:52:22.605 INFO:journalctl@ceph.mon.c.smithi059.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:52:22.605 INFO:journalctl@ceph.mon.c.smithi059.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T15:52:22.605 INFO:journalctl@ceph.mon.c.smithi059.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T15:52:22.605 INFO:journalctl@ceph.mon.c.smithi059.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:52:22.605 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T15:52:22.605 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T15:52:22.605 INFO:journalctl@ceph.mon.c.smithi059.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T15:52:22.606 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: Interval compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T15:52:22.607 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T15:52:22.608 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: using public_addrv [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: starting mon.c rank -1 at public addrs [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] at bind addrs [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(???) e0 preinit fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).mds e1 new map 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout: e1 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout: legacy client fscid: -1 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout: No filesystems configured 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-26T15:52:22.609 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi059:172.21.15.59=a;smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c;smithi064:172.21.15.64=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: Saving service mon spec with placement smithi059:172.21.15.59=a;smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c;smithi064:172.21.15.64=b;count:3 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: Deploying daemon mon.c on smithi059 2024-04-26T15:52:22.610 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:22.611 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/1109922982' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:22.611 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-26T15:52:22.611 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 ceph-mon[26536]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-26T15:52:22.611 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 bash[26518]: 17313040ebe2f77c28d4531a3a9e89a1fa9acef98a9f00e4424d1ceba67fe2a4 2024-04-26T15:52:22.611 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:22 smithi059 systemd[1]: Started Ceph mon.c for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:52:22.788 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T15:52:22.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:52:27.552 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-04-26T15:52:27.552 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":2,"fsid":"b6607c98-03e4-11ef-bc93-c7b262605968","modified":"2024-04-26T15:52:22.254799Z","created":"2024-04-26T15:50:59.741700Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3300","nonce":0},{"type":"v1","addr":"172.21.15.59:6789","nonce":0}]},"addr":"172.21.15.59:6789/0","public_addr":"172.21.15.59:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3301","nonce":0},{"type":"v1","addr":"172.21.15.59:6790","nonce":0}]},"addr":"172.21.15.59:6790/0","public_addr":"172.21.15.59:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-26T15:52:27.554 INFO:teuthology.orchestra.run.smithi064.stderr:dumped monmap epoch 2 2024-04-26T15:52:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:52:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: mon.a calling monitor election 2024-04-26T15:52:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: mon.c calling monitor election 2024-04-26T15:52:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: monmap e2: 2 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: fsmap 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: mgrmap e13: y(active, since 35s) 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: overall HEALTH_OK 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:27.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: mon.a calling monitor election 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: mon.c calling monitor election 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: monmap e2: 2 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T15:52:27.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: fsmap 2024-04-26T15:52:27.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:52:27.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: mgrmap e13: y(active, since 35s) 2024-04-26T15:52:27.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: overall HEALTH_OK 2024-04-26T15:52:27.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:27.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:27.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:28.771 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:28 smithi059 ceph-mon[26536]: Deploying daemon mon.b on smithi064 2024-04-26T15:52:28.771 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:28 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/759149990' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:28.771 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:28 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:28.771 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:28 smithi059 ceph-mon[21819]: Deploying daemon mon.b on smithi064 2024-04-26T15:52:28.771 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:28 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/759149990' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:28.771 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:28 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:29.100 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T15:52:29.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:52:29.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:29 smithi059 ceph-mon[21819]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:29 smithi059 ceph-mon[26536]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-04-26T15:52:31.177 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":2,"fsid":"b6607c98-03e4-11ef-bc93-c7b262605968","modified":"2024-04-26T15:52:22.254799Z","created":"2024-04-26T15:50:59.741700Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3300","nonce":0},{"type":"v1","addr":"172.21.15.59:6789","nonce":0}]},"addr":"172.21.15.59:6789/0","public_addr":"172.21.15.59:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3301","nonce":0},{"type":"v1","addr":"172.21.15.59:6790","nonce":0}]},"addr":"172.21.15.59:6790/0","public_addr":"172.21.15.59:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-26T15:52:31.178 INFO:teuthology.orchestra.run.smithi064.stderr:dumped monmap epoch 2 2024-04-26T15:52:31.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:31 smithi059 ceph-mon[21819]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:31.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:31 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/3376375286' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:31 smithi059 ceph-mon[26536]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:31 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/3376375286' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:32.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 podman[22724]: 2024-04-26 15:52:32.443006392 +0000 UTC m=+0.126427899 container create cb2d5179017db3932cd46a59cb85a502f6206767eb9752f5e95f921b12b52c55 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, release=754, version=8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-type=git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-26T15:52:32.744 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 podman[22724]: 2024-04-26 15:52:32.628533608 +0000 UTC m=+0.311955115 container init cb2d5179017db3932cd46a59cb85a502f6206767eb9752f5e95f921b12b52c55 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, distribution-scope=public, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vcs-type=git, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-26T15:52:32.745 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 podman[22724]: 2024-04-26 15:52:32.631426851 +0000 UTC m=+0.314848358 container start cb2d5179017db3932cd46a59cb85a502f6206767eb9752f5e95f921b12b52c55 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, architecture=x86_64, RELEASE=HEAD, vendor=Red Hat, Inc., version=8, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-26T15:52:32.745 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T15:52:32.745 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-26T15:52:32.745 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: pidfile_write: ignore empty --pid-file 2024-04-26T15:52:32.745 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: load: jerasure load: lrc 2024-04-26T15:52:32.745 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: RocksDB version: 6.15.5 2024-04-26T15:52:32.745 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T15:52:32.745 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Compile date Apr 18 2022 2024-04-26T15:52:32.745 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: DB SUMMARY 2024-04-26T15:52:32.745 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: DB Session ID: QC2EAF9Y1TRALPS13PK4 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: CURRENT file: CURRENT 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.error_if_exists: 0 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.create_if_missing: 0 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.paranoid_checks: 1 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.env: 0x55f13b4c6860 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.fs: Posix File System 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.info_log: 0x55f13d7e7dc0 2024-04-26T15:52:32.746 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.statistics: (nil) 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.use_fsync: 0 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_log_file_size: 0 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.allow_fallocate: 1 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T15:52:32.747 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.use_direct_reads: 0 2024-04-26T15:52:32.748 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T15:52:32.748 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T15:52:32.748 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.db_log_dir: 2024-04-26T15:52:32.748 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-26T15:52:32.748 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T15:52:32.748 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T15:52:32.748 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T15:52:32.748 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T15:52:32.748 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T15:52:32.748 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T15:52:32.749 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T15:52:32.749 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.write_buffer_manager: 0x55f13d8d8240 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.unordered_write: 0 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T15:52:32.750 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.row_cache: None 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.wal_filter: None 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.preserve_deletes: 0 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.two_write_queues: 0 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.atomic_flush: 0 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.log_readahead_size: 0 2024-04-26T15:52:32.751 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_background_jobs: 2 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_background_compactions: -1 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_subcompactions: 1 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T15:52:32.752 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_open_files: -1 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_background_flushes: -1 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Compression algorithms supported: 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: kZSTD supported: 0 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: kXpressCompression supported: 0 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: kLZ4Compression supported: 1 2024-04-26T15:52:32.753 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: kBZip2Compression supported: 0 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: kZlibCompression supported: 1 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: kSnappyCompression supported: 1 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.merge_operator: 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_filter: None 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T15:52:32.754 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f13d7b3d00) 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: cache_index_and_filter_blocks: 1 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: pin_top_level_index_and_filter: 1 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_type: 0 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: data_block_index_type: 0 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_shortening: 1 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: hash_index_allow_collision: 1 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: checksum: 1 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: no_block_cache: 0 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache: 0x55f13d81e170 2024-04-26T15:52:32.755 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_name: BinnedLRUCache 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_options: 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: capacity : 536870912 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: num_shard_bits : 4 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: strict_capacity_limit : 0 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: high_pri_pool_ratio: 0.000 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_compressed: (nil) 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: persistent_cache: (nil) 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_size: 4096 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_size_deviation: 10 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_restart_interval: 16 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_block_restart_interval: 1 2024-04-26T15:52:32.756 INFO:journalctl@ceph.mon.b.smithi064.stdout: metadata_block_size: 4096 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout: partition_filters: 0 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_delta_encoding: 1 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout: whole_key_filtering: 1 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout: verify_compression: 0 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout: read_amp_bytes_per_bit: 0 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout: format_version: 4 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout: enable_index_compression: 1 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_align: 0 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compression: NoCompression 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T15:52:32.757 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.num_levels: 7 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:52:32.758 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T15:52:32.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T15:52:32.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.table_properties_collectors: 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.inplace_update_support: 0 2024-04-26T15:52:32.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.bloom_locality: 0 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.max_successive_merges: 0 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.ttl: 2592000 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.enable_blob_files: false 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.min_blob_size: 0 2024-04-26T15:52:32.762 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T15:52:32.763 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T15:52:32.763 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T15:52:32.763 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T15:52:32.763 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T15:52:32.763 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-26T15:52:32.763 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-26T15:52:32.763 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714146752658129, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-26T15:52:32.763 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-26T15:52:32.763 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T15:52:32.763 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714146752658435, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714146752, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "effdba0f-4e95-446a-aa18-349fcc718a59", "db_session_id": "QC2EAF9Y1TRALPS13PK4"}} 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714146752658753, "job": 1, "event": "recovery_finished"} 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55f13d804a80 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: DB pointer 0x55f13d814000 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** DB Stats ** 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T15:52:32.764 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:52:32.765 INFO:journalctl@ceph.mon.b.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T15:52:32.766 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T15:52:32.767 INFO:journalctl@ceph.mon.b.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-26T15:52:32.768 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: using public_addr v2:172.21.15.64:0/0 -> [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] 2024-04-26T15:52:32.769 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: starting mon.b rank -1 at public addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] at bind addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:52:32.769 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(???) e0 preinit fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:52:32.769 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).mds e1 new map 2024-04-26T15:52:32.769 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-26T15:52:32.769 INFO:journalctl@ceph.mon.b.smithi064.stdout: e1 2024-04-26T15:52:32.769 INFO:journalctl@ceph.mon.b.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T15:52:32.770 INFO:journalctl@ceph.mon.b.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T15:52:32.770 INFO:journalctl@ceph.mon.b.smithi064.stdout: legacy client fscid: -1 2024-04-26T15:52:32.770 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T15:52:32.770 INFO:journalctl@ceph.mon.b.smithi064.stdout: No filesystems configured 2024-04-26T15:52:32.770 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-26T15:52:32.770 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-26T15:52:32.770 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-26T15:52:32.770 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-26T15:52:32.770 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi059:172.21.15.59=a;smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c;smithi064:172.21.15.64=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: Saving service mon spec with placement smithi059:172.21.15.59=a;smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c;smithi064:172.21.15.64=b;count:3 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: Deploying daemon mon.c on smithi059 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/1109922982' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:32.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.a calling monitor election 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.c calling monitor election 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: monmap e2: 2 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T15:52:32.772 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: fsmap 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mgrmap e13: y(active, since 35s) 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: overall HEALTH_OK 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: Deploying daemon mon.b on smithi064 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/759149990' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/3376375286' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing).paxosservice(auth 1..4) refresh upgraded, format 0 -> 3 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 ceph-mon[22742]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-26T15:52:32.773 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 bash[22724]: cb2d5179017db3932cd46a59cb85a502f6206767eb9752f5e95f921b12b52c55 2024-04-26T15:52:32.774 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:32 smithi064 systemd[1]: Started Ceph mon.b for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:52:33.089 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T15:52:33.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T15:52:37.974 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-04-26T15:52:37.974 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":3,"fsid":"b6607c98-03e4-11ef-bc93-c7b262605968","modified":"2024-04-26T15:52:32.680547Z","created":"2024-04-26T15:50:59.741700Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3300","nonce":0},{"type":"v1","addr":"172.21.15.59:6789","nonce":0}]},"addr":"172.21.15.59:6789/0","public_addr":"172.21.15.59:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3301","nonce":0},{"type":"v1","addr":"172.21.15.59:6790","nonce":0}]},"addr":"172.21.15.59:6790/0","public_addr":"172.21.15.59:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-26T15:52:37.976 INFO:teuthology.orchestra.run.smithi064.stderr:dumped monmap epoch 3 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: mon.a calling monitor election 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: mon.c calling monitor election 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: fsmap 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: mgrmap e13: y(active, since 45s) 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: mon.b (rank 2) addr [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] is down (out of quorum) 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: mon.a calling monitor election 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: mon.c calling monitor election 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-26T15:52:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: fsmap 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: mgrmap e13: y(active, since 45s) 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: mon.b (rank 2) addr [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] is down (out of quorum) 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:38.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:37 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:38.521 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-26T15:52:38.521 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-26T15:52:38.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:38 smithi059 ceph-mon[26536]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:38.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:38 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/4041554963' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:38.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:38 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:38.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:38 smithi059 ceph-mon[21819]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:38.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:38 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/4041554963' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:38.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:38 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: mon.a calling monitor election 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: mon.c calling monitor election 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: fsmap 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: mgrmap e13: y(active, since 45s) 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: mon.b (rank 2) addr [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] is down (out of quorum) 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/4041554963' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T15:52:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:40.492 INFO:teuthology.orchestra.run.smithi059.stdout:# minimal ceph.conf for b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:52:40.492 INFO:teuthology.orchestra.run.smithi059.stdout:[global] 2024-04-26T15:52:40.492 INFO:teuthology.orchestra.run.smithi059.stdout: fsid = b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T15:52:40.492 INFO:teuthology.orchestra.run.smithi059.stdout: mon_host = [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] 2024-04-26T15:52:40.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: mon.b calling monitor election 2024-04-26T15:52:40.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: mon.b calling monitor election 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: mon.c calling monitor election 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: mon.a calling monitor election 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: fsmap 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: mgrmap e13: y(active, since 47s) 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: Cluster is now healthy 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: overall HEALTH_OK 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:40.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: mon.b calling monitor election 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: mon.b calling monitor election 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: mon.c calling monitor election 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: mon.a calling monitor election 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: fsmap 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: mgrmap e13: y(active, since 47s) 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: Cluster is now healthy 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: overall HEALTH_OK 2024-04-26T15:52:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:40 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: mon.b calling monitor election 2024-04-26T15:52:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: mon.b calling monitor election 2024-04-26T15:52:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: mon.c calling monitor election 2024-04-26T15:52:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: mon.a calling monitor election 2024-04-26T15:52:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T15:52:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T15:52:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: fsmap 2024-04-26T15:52:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T15:52:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: mgrmap e13: y(active, since 47s) 2024-04-26T15:52:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-26T15:52:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: Cluster is now healthy 2024-04-26T15:52:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: overall HEALTH_OK 2024-04-26T15:52:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:40 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:41.045 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-26T15:52:41.045 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-26T15:52:41.045 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T15:52:41.073 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-26T15:52:41.073 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:52:41.139 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-04-26T15:52:41.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T15:52:41.169 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-04-26T15:52:41.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:52:41.238 INFO:tasks.cephadm:Adding mgr.y on smithi059 2024-04-26T15:52:41.238 INFO:tasks.cephadm:Adding mgr.x on smithi064 2024-04-26T15:52:41.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi059=y;smithi064=x' 2024-04-26T15:52:41.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:41 smithi064 ceph-mon[22742]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:41.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:41 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T15:52:41.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:41 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1862136472' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:41 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:41 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:41 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:41 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:52:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:41 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:41 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:41.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[21819]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:41.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T15:52:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1862136472' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:52:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[26536]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T15:52:41.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1862136472' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:41.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:41.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:41.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:41.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:52:41.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:41.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:41 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:52:42.507 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[21819]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T15:52:42.507 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[21819]: Reconfiguring daemon mon.a on smithi059 2024-04-26T15:52:42.507 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:42.507 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:42.507 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:42.507 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:52:42.507 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:42.508 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[26536]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T15:52:42.508 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[26536]: Reconfiguring daemon mon.a on smithi059 2024-04-26T15:52:42.508 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:42.508 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:42.508 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:42.508 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:52:42.508 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:42 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:42.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:42 smithi064 ceph-mon[22742]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T15:52:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:42 smithi064 ceph-mon[22742]: Reconfiguring daemon mon.a on smithi059 2024-04-26T15:52:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:42 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:42 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:42 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:42 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:52:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:42 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:43.090 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled mgr update... 2024-04-26T15:52:43.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:43 smithi059 ceph-mon[21819]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:43.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:43 smithi059 ceph-mon[21819]: Reconfiguring mon.c (monmap changed)... 2024-04-26T15:52:43.459 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:43 smithi059 ceph-mon[21819]: Reconfiguring daemon mon.c on smithi059 2024-04-26T15:52:43.459 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:43 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:43.459 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:43 smithi059 ceph-mon[26536]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:43.459 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:43 smithi059 ceph-mon[26536]: Reconfiguring mon.c (monmap changed)... 2024-04-26T15:52:43.459 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:43 smithi059 ceph-mon[26536]: Reconfiguring daemon mon.c on smithi059 2024-04-26T15:52:43.459 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:43 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:43.588 DEBUG:teuthology.orchestra.run.smithi064:mgr.x> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service 2024-04-26T15:52:43.591 INFO:tasks.cephadm:Deploying OSDs... 2024-04-26T15:52:43.591 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-26T15:52:43.591 DEBUG:teuthology.orchestra.run.smithi059:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T15:52:43.609 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T15:52:43.609 DEBUG:teuthology.orchestra.run.smithi059:> stat /dev/vg_nvme/lv_1 2024-04-26T15:52:43.610 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:43 smithi064 ceph-mon[22742]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:43.610 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:43 smithi064 ceph-mon[22742]: Reconfiguring mon.c (monmap changed)... 2024-04-26T15:52:43.610 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:43 smithi064 ceph-mon[22742]: Reconfiguring daemon mon.c on smithi059 2024-04-26T15:52:43.610 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:43 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:43.667 INFO:teuthology.orchestra.run.smithi059.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T15:52:43.667 INFO:teuthology.orchestra.run.smithi059.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:52:43.667 INFO:teuthology.orchestra.run.smithi059.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-04-26T15:52:43.667 INFO:teuthology.orchestra.run.smithi059.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:52:43.667 INFO:teuthology.orchestra.run.smithi059.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:52:43.667 INFO:teuthology.orchestra.run.smithi059.stdout:Access: 2024-04-26 15:52:21.706234695 +0000 2024-04-26T15:52:43.667 INFO:teuthology.orchestra.run.smithi059.stdout:Modify: 2024-04-26 15:50:10.852090761 +0000 2024-04-26T15:52:43.667 INFO:teuthology.orchestra.run.smithi059.stdout:Change: 2024-04-26 15:50:10.852090761 +0000 2024-04-26T15:52:43.667 INFO:teuthology.orchestra.run.smithi059.stdout: Birth: 2024-04-26 15:50:10.852090761 +0000 2024-04-26T15:52:43.667 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T15:52:43.734 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records in 2024-04-26T15:52:43.734 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records out 2024-04-26T15:52:43.734 INFO:teuthology.orchestra.run.smithi059.stderr:512 bytes copied, 0.000220078 s, 2.3 MB/s 2024-04-26T15:52:43.735 DEBUG:teuthology.orchestra.run.smithi059:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T15:52:43.793 DEBUG:teuthology.orchestra.run.smithi059:> stat /dev/vg_nvme/lv_2 2024-04-26T15:52:43.850 INFO:teuthology.orchestra.run.smithi059.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T15:52:43.850 INFO:teuthology.orchestra.run.smithi059.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:52:43.850 INFO:teuthology.orchestra.run.smithi059.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-26T15:52:43.850 INFO:teuthology.orchestra.run.smithi059.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:52:43.850 INFO:teuthology.orchestra.run.smithi059.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:52:43.850 INFO:teuthology.orchestra.run.smithi059.stdout:Access: 2024-04-26 15:52:21.706234695 +0000 2024-04-26T15:52:43.850 INFO:teuthology.orchestra.run.smithi059.stdout:Modify: 2024-04-26 15:50:11.229085786 +0000 2024-04-26T15:52:43.851 INFO:teuthology.orchestra.run.smithi059.stdout:Change: 2024-04-26 15:50:11.229085786 +0000 2024-04-26T15:52:43.851 INFO:teuthology.orchestra.run.smithi059.stdout: Birth: 2024-04-26 15:50:11.229085786 +0000 2024-04-26T15:52:43.851 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T15:52:43.915 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records in 2024-04-26T15:52:43.915 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records out 2024-04-26T15:52:43.915 INFO:teuthology.orchestra.run.smithi059.stderr:512 bytes copied, 0.000222177 s, 2.3 MB/s 2024-04-26T15:52:43.916 DEBUG:teuthology.orchestra.run.smithi059:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T15:52:43.973 DEBUG:teuthology.orchestra.run.smithi059:> stat /dev/vg_nvme/lv_3 2024-04-26T15:52:44.029 INFO:teuthology.orchestra.run.smithi059.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T15:52:44.029 INFO:teuthology.orchestra.run.smithi059.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:52:44.030 INFO:teuthology.orchestra.run.smithi059.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-26T15:52:44.030 INFO:teuthology.orchestra.run.smithi059.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:52:44.030 INFO:teuthology.orchestra.run.smithi059.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:52:44.030 INFO:teuthology.orchestra.run.smithi059.stdout:Access: 2024-04-26 15:52:21.707234683 +0000 2024-04-26T15:52:44.030 INFO:teuthology.orchestra.run.smithi059.stdout:Modify: 2024-04-26 15:50:11.588081048 +0000 2024-04-26T15:52:44.030 INFO:teuthology.orchestra.run.smithi059.stdout:Change: 2024-04-26 15:50:11.588081048 +0000 2024-04-26T15:52:44.030 INFO:teuthology.orchestra.run.smithi059.stdout: Birth: 2024-04-26 15:50:11.588081048 +0000 2024-04-26T15:52:44.030 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T15:52:44.093 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records in 2024-04-26T15:52:44.094 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records out 2024-04-26T15:52:44.094 INFO:teuthology.orchestra.run.smithi059.stderr:512 bytes copied, 0.000231186 s, 2.2 MB/s 2024-04-26T15:52:44.095 DEBUG:teuthology.orchestra.run.smithi059:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T15:52:44.152 DEBUG:teuthology.orchestra.run.smithi059:> stat /dev/vg_nvme/lv_4 2024-04-26T15:52:44.208 INFO:teuthology.orchestra.run.smithi059.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T15:52:44.208 INFO:teuthology.orchestra.run.smithi059.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:52:44.208 INFO:teuthology.orchestra.run.smithi059.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-26T15:52:44.209 INFO:teuthology.orchestra.run.smithi059.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:52:44.209 INFO:teuthology.orchestra.run.smithi059.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:52:44.209 INFO:teuthology.orchestra.run.smithi059.stdout:Access: 2024-04-26 15:52:21.707234683 +0000 2024-04-26T15:52:44.209 INFO:teuthology.orchestra.run.smithi059.stdout:Modify: 2024-04-26 15:50:11.927076574 +0000 2024-04-26T15:52:44.209 INFO:teuthology.orchestra.run.smithi059.stdout:Change: 2024-04-26 15:50:11.927076574 +0000 2024-04-26T15:52:44.209 INFO:teuthology.orchestra.run.smithi059.stdout: Birth: 2024-04-26 15:50:11.927076574 +0000 2024-04-26T15:52:44.209 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T15:52:44.272 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records in 2024-04-26T15:52:44.272 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records out 2024-04-26T15:52:44.272 INFO:teuthology.orchestra.run.smithi059.stderr:512 bytes copied, 0.000239493 s, 2.1 MB/s 2024-04-26T15:52:44.273 DEBUG:teuthology.orchestra.run.smithi059:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T15:52:44.330 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-04-26T15:52:44.330 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T15:52:44.347 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T15:52:44.347 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_1 2024-04-26T15:52:44.403 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T15:52:44.403 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:52:44.404 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-26T15:52:44.404 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:52:44.404 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:52:44.404 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-04-26 15:52:32.128335721 +0000 2024-04-26T15:52:44.404 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-04-26 15:50:10.857988497 +0000 2024-04-26T15:52:44.404 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-04-26 15:50:10.857988497 +0000 2024-04-26T15:52:44.404 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-04-26 15:50:10.857988497 +0000 2024-04-26T15:52:44.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T15:52:44.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:44 smithi064 ceph-mon[22742]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi059=y;smithi064=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:44.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:44 smithi064 ceph-mon[22742]: Saving service mgr spec with placement smithi059=y;smithi064=x;count:2 2024-04-26T15:52:44.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:44 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:44.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:44 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:44.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:44 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:52:44.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:44 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-04-26T15:52:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-04-26T15:52:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000212802 s, 2.4 MB/s 2024-04-26T15:52:44.462 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T15:52:44.519 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_2 2024-04-26T15:52:44.575 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T15:52:44.575 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:52:44.575 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-26T15:52:44.575 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:52:44.575 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:52:44.576 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-04-26 15:52:32.129335701 +0000 2024-04-26T15:52:44.576 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-04-26 15:50:11.207980708 +0000 2024-04-26T15:52:44.576 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-04-26 15:50:11.207980708 +0000 2024-04-26T15:52:44.576 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-04-26 15:50:11.207980708 +0000 2024-04-26T15:52:44.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T15:52:44.638 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-04-26T15:52:44.638 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-04-26T15:52:44.638 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000198544 s, 2.6 MB/s 2024-04-26T15:52:44.639 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T15:52:44.695 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_3 2024-04-26T15:52:44.750 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T15:52:44.750 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:52:44.750 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-26T15:52:44.751 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:52:44.751 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:52:44.751 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-04-26 15:52:32.129335701 +0000 2024-04-26T15:52:44.751 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-04-26 15:50:11.566972722 +0000 2024-04-26T15:52:44.751 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-04-26 15:50:11.566972722 +0000 2024-04-26T15:52:44.751 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-04-26 15:50:11.566972722 +0000 2024-04-26T15:52:44.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T15:52:44.814 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-04-26T15:52:44.814 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-04-26T15:52:44.815 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000206553 s, 2.5 MB/s 2024-04-26T15:52:44.816 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T15:52:44.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[21819]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi059=y;smithi064=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:44.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[21819]: Saving service mgr spec with placement smithi059=y;smithi064=x;count:2 2024-04-26T15:52:44.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:44.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:44.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:52:44.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[26536]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi059=y;smithi064=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[26536]: Saving service mgr spec with placement smithi059=y;smithi064=x;count:2 2024-04-26T15:52:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T15:52:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T15:52:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:44 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:44.872 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_4 2024-04-26T15:52:44.927 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T15:52:44.927 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T15:52:44.928 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-26T15:52:44.928 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T15:52:44.928 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T15:52:44.928 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-04-26 15:52:32.130335681 +0000 2024-04-26T15:52:44.928 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-04-26 15:50:11.907965135 +0000 2024-04-26T15:52:44.928 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-04-26 15:50:11.907965135 +0000 2024-04-26T15:52:44.928 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-04-26 15:50:11.907965135 +0000 2024-04-26T15:52:44.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T15:52:44.993 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-04-26T15:52:44.993 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-04-26T15:52:44.993 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000226466 s, 2.3 MB/s 2024-04-26T15:52:44.994 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T15:52:45.051 INFO:tasks.cephadm:Deploying osd.0 on smithi059 with /dev/vg_nvme/lv_4... 2024-04-26T15:52:45.051 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T15:52:45.636 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: Reconfiguring mon.b (monmap changed)... 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: Reconfiguring daemon mon.b on smithi064 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:52:45.637 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: Reconfiguring mon.b (monmap changed)... 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: Reconfiguring daemon mon.b on smithi064 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:52:45.638 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: Reconfiguring mon.b (monmap changed)... 2024-04-26T15:52:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: Reconfiguring daemon mon.b on smithi064 2024-04-26T15:52:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:45.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:45.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:45.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:45.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:45.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:52:45.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T15:52:45.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:52:45.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:46.500 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:46 smithi059 ceph-mon[21819]: Deploying daemon mgr.x on smithi064 2024-04-26T15:52:46.500 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:46 smithi059 ceph-mon[26536]: Deploying daemon mgr.x on smithi064 2024-04-26T15:52:46.603 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:46 smithi064 ceph-mon[22742]: Deploying daemon mgr.x on smithi064 2024-04-26T15:52:46.603 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:46 smithi064 podman[23907]: 2024-04-26 15:52:46.604048461 +0000 UTC m=+0.146141277 container create 06cc1766b70cc044370150ae95baae1706deecea43a03451ceb11a53fa6406c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., architecture=x86_64, distribution-scope=public, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, release=754, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-26T15:52:46.922 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:46 smithi064 podman[23907]: 2024-04-26 15:52:46.814850472 +0000 UTC m=+0.356943298 container init 06cc1766b70cc044370150ae95baae1706deecea43a03451ceb11a53fa6406c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, RELEASE=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., ceph=True, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, architecture=x86_64, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.buildah.version=1.19.8) 2024-04-26T15:52:46.922 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:46 smithi064 podman[23907]: 2024-04-26 15:52:46.81762193 +0000 UTC m=+0.359714746 container start 06cc1766b70cc044370150ae95baae1706deecea43a03451ceb11a53fa6406c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux ) 2024-04-26T15:52:46.922 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:46 smithi064 ceph-mgr[23926]: -- 172.21.15.64:0/4267432735 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d148c0a340 con 0x55d148be2c00 2024-04-26T15:52:46.922 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:46 smithi064 bash[23907]: 06cc1766b70cc044370150ae95baae1706deecea43a03451ceb11a53fa6406c8 2024-04-26T15:52:47.009 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T15:52:47.009 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T15:52:47.009 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10+0 records in 2024-04-26T15:52:47.009 INFO:teuthology.orchestra.run.smithi059.stderr:10+0 records out 2024-04-26T15:52:47.009 INFO:teuthology.orchestra.run.smithi059.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0114851 s, 913 MB/s 2024-04-26T15:52:47.009 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 2024-04-26T15:52:47.009 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping successful for: 2024-04-26T15:52:47.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:46 smithi064 systemd[1]: Started Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:52:47.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:46 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:46.972+0000 7f5a83812000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T15:52:47.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:47 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:47.110+0000 7f5a83812000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:52:47.535 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:47 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:47.215+0000 7f5a83812000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T15:52:47.536 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:47 smithi064 ceph-mon[22742]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:47.536 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:47.536 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:47.536 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:47.536 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:47.629 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:47 smithi059 ceph-mon[21819]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:47.629 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:47.630 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:47.630 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:47.630 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:47.630 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:47 smithi059 ceph-mon[26536]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:47.630 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:47.630 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:47.630 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:47.630 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:47.657 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi059:vg_nvme/lv_4 2024-04-26T15:52:47.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:47 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:47.560+0000 7f5a83812000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T15:52:47.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:47 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:47.648+0000 7f5a83812000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T15:52:49.105 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:48.852+0000 7f5a83812000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T15:52:49.105 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:49 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:49.021+0000 7f5a83812000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T15:52:49.106 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:49 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:49.106+0000 7f5a83812000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T15:52:49.106 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:48 smithi064 ceph-mon[22742]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:49.106 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:48 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:49.106 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:48 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:49.106 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:48 smithi064 ceph-mon[22742]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T15:52:49.106 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:48 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:52:49.106 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:48 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:52:49.106 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:48 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:49.106 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:48 smithi064 ceph-mon[22742]: Reconfiguring daemon mgr.y on smithi059 2024-04-26T15:52:49.263 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[21819]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:49.263 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:49.263 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:49.263 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[21819]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T15:52:49.263 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:52:49.263 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:52:49.263 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:49.263 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[21819]: Reconfiguring daemon mgr.y on smithi059 2024-04-26T15:52:49.263 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[26536]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:49.263 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:49.264 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:49.264 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[26536]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T15:52:49.264 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T15:52:49.264 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T15:52:49.264 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:49.264 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:48 smithi059 ceph-mon[26536]: Reconfiguring daemon mgr.y on smithi059 2024-04-26T15:52:49.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:49 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:49.427+0000 7f5a83812000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T15:52:49.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:49 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:49.520+0000 7f5a83812000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T15:52:49.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:49 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:49.616+0000 7f5a83812000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T15:52:50.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:49 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:49.863+0000 7f5a83812000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:52:50.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:50.324+0000 7f5a83812000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T15:52:51.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:50.824+0000 7f5a83812000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T15:52:51.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:50.921+0000 7f5a83812000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T15:52:51.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[21819]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:51.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:51.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[26536]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:52:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:52:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:51.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:51.224+0000 7f5a83812000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T15:52:51.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:51.316+0000 7f5a83812000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T15:52:51.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-mon[22742]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:52:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:52:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:52:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:52:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:52.080 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:51.730+0000 7f5a83812000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T15:52:52.080 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:51.867+0000 7f5a83812000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T15:52:52.080 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:51.962+0000 7f5a83812000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T15:52:52.080 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:52.079+0000 7f5a83812000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T15:52:52.224 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:52 smithi059 ceph-mon[26536]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:52.224 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:52 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:52:52.224 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:52 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:52.224 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:52 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:52:52.224 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:52 smithi059 ceph-mon[21819]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:52.224 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:52:52.224 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:52.225 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:52:52.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:52.171+0000 7f5a83812000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T15:52:52.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:52 smithi064 ceph-mon[22742]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:52:52.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:52 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:52:52.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:52 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:52:52.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:52 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:52:53.110 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:52.739+0000 7f5a83812000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T15:52:53.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:52:53.109+0000 7f5a83812000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T15:52:53.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:53.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3252865284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda"}]: dispatch 2024-04-26T15:52:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda"}]: dispatch 2024-04-26T15:52:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda"}]': finished 2024-04-26T15:52:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T15:52:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:52:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: Standby manager daemon x started 2024-04-26T15:52:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T15:52:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:52:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T15:52:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:53 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:52:53.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3252865284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda"}]: dispatch 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda"}]: dispatch 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda"}]': finished 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: Standby manager daemon x started 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:52:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3252865284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda"}]: dispatch 2024-04-26T15:52:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda"}]: dispatch 2024-04-26T15:52:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda"}]': finished 2024-04-26T15:52:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T15:52:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:52:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: Standby manager daemon x started 2024-04-26T15:52:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T15:52:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:52:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T15:52:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:53 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/1257613015' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:52:54.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:54 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3469222309' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:52:54.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:54 smithi059 ceph-mon[21819]: mgrmap e14: y(active, since 62s), standbys: x 2024-04-26T15:52:54.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:54 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T15:52:54.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:54 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3469222309' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:52:54.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:54 smithi059 ceph-mon[26536]: mgrmap e14: y(active, since 62s), standbys: x 2024-04-26T15:52:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:54 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T15:52:54.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:54 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3469222309' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:52:54.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:54 smithi064 ceph-mon[22742]: mgrmap e14: y(active, since 62s), standbys: x 2024-04-26T15:52:54.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:54 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T15:52:55.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:55 smithi059 ceph-mon[26536]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:55.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:55 smithi059 ceph-mon[21819]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:55.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:55 smithi064 ceph-mon[22742]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:57.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:57 smithi059 ceph-mon[21819]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:57.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:57 smithi059 ceph-mon[26536]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:57 smithi064 ceph-mon[22742]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:59.464 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:59 smithi059 ceph-mon[21819]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:59.464 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:59 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T15:52:59.464 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:52:59 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:59.465 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:59 smithi059 ceph-mon[26536]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:59.465 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:59 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T15:52:59.465 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:52:59 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:52:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:59 smithi064 ceph-mon[22742]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:52:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:59 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T15:52:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:52:59 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:00.560 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:00 smithi059 ceph-mon[21819]: Deploying daemon osd.0 on smithi059 2024-04-26T15:53:00.561 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:00 smithi059 ceph-mon[26536]: Deploying daemon osd.0 on smithi059 2024-04-26T15:53:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:00 smithi064 ceph-mon[22742]: Deploying daemon osd.0 on smithi059 2024-04-26T15:53:01.457 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:01 smithi059 ceph-mon[21819]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:01.458 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:01 smithi059 ceph-mon[26536]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:01.587 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:01 smithi064 ceph-mon[22742]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:03.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:03 smithi059 ceph-mon[21819]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:03.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:03 smithi059 ceph-mon[26536]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:03.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:03 smithi064 ceph-mon[22742]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:04.558 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:04 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:04.558 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:04 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:04.558 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:04 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:04.558 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:04 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:04.559 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:04 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:04.559 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:04 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:04.559 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:04 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:04.559 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:04 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:04.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:04 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:04.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:04 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:04.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:04 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:04.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:04 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:05.491 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:05 smithi059 ceph-mon[21819]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:05.491 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:05 smithi059 ceph-mon[26536]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:05.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:05 smithi064 ceph-mon[22742]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:06.938 INFO:teuthology.orchestra.run.smithi059.stdout:Created osd(s) 0 on host 'smithi059' 2024-04-26T15:53:07.508 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:07 smithi059 ceph-mon[21819]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:07.509 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:07.509 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:07 smithi059 ceph-mon[26536]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:07.509 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:07.510 DEBUG:teuthology.orchestra.run.smithi059:osd.0> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.0.service 2024-04-26T15:53:07.512 INFO:tasks.cephadm:Deploying osd.1 on smithi059 with /dev/vg_nvme/lv_3... 2024-04-26T15:53:07.512 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T15:53:07.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:07 smithi064 ceph-mon[22742]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:07.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:08.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:08 smithi059 ceph-mon[26536]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T15:53:08.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:08 smithi059 ceph-mon[21819]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T15:53:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:08 smithi064 ceph-mon[22742]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T15:53:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-mon[21819]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-mon[21819]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T15:53:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-mon[21819]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T15:53:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-mon[21819]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:53:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-mon[26536]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-mon[26536]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T15:53:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-mon[26536]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T15:53:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-mon[26536]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:53:09.583 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[29410]: 2024-04-26T15:53:09.274+0000 7fd0c2ae1700 -1 osd.0 0 waiting for initial osdmap 2024-04-26T15:53:09.583 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 15:53:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[29410]: 2024-04-26T15:53:09.278+0000 7fd0bf47c700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:53:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:09 smithi064 ceph-mon[22742]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:09 smithi064 ceph-mon[22742]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T15:53:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:09 smithi064 ceph-mon[22742]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T15:53:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:09 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:09.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:09 smithi064 ceph-mon[22742]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:53:09.772 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T15:53:09.772 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T15:53:09.772 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10+0 records in 2024-04-26T15:53:09.772 INFO:teuthology.orchestra.run.smithi059.stderr:10+0 records out 2024-04-26T15:53:09.773 INFO:teuthology.orchestra.run.smithi059.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0152974 s, 685 MB/s 2024-04-26T15:53:09.773 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping successful for: 2024-04-26T15:53:10.406 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi059:vg_nvme/lv_3 2024-04-26T15:53:10.567 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:10 smithi059 ceph-mon[26536]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:53:10.568 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:10 smithi059 ceph-mon[26536]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T15:53:10.568 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:10 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:10.568 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:10 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:10.568 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:10 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:10.568 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:10 smithi059 ceph-mon[21819]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:53:10.568 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:10 smithi059 ceph-mon[21819]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T15:53:10.568 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:10 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:10.568 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:10 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:10.568 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:10 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:10.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:10 smithi064 ceph-mon[22742]: from='osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:53:10.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:10 smithi064 ceph-mon[22742]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T15:53:10.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:10 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:10.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:10 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:10.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:10 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:11.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[26536]: purged_snaps scrub starts 2024-04-26T15:53:11.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[26536]: purged_snaps scrub ok 2024-04-26T15:53:11.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[26536]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:11.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:11.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[26536]: osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871] boot 2024-04-26T15:53:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[26536]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T15:53:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[21819]: purged_snaps scrub starts 2024-04-26T15:53:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[21819]: purged_snaps scrub ok 2024-04-26T15:53:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[21819]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[21819]: osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871] boot 2024-04-26T15:53:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[21819]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T15:53:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:11 smithi064 ceph-mon[22742]: purged_snaps scrub starts 2024-04-26T15:53:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:11 smithi064 ceph-mon[22742]: purged_snaps scrub ok 2024-04-26T15:53:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:11 smithi064 ceph-mon[22742]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T15:53:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:11 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:11 smithi064 ceph-mon[22742]: osd.0 [v2:172.21.15.59:6802/1726437871,v1:172.21.15.59:6803/1726437871] boot 2024-04-26T15:53:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:11 smithi064 ceph-mon[22742]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T15:53:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:11 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:53:12.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:12 smithi059 ceph-mon[21819]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T15:53:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:12 smithi059 ceph-mon[26536]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T15:53:12.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:12 smithi064 ceph-mon[22742]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T15:53:13.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:13 smithi059 ceph-mon[26536]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:13.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:13 smithi059 ceph-mon[26536]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:53:13.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:13 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:53:13.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:13 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:53:13.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:13 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:13 smithi059 ceph-mon[21819]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:13 smithi059 ceph-mon[21819]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:53:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:13 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:53:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:13 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:53:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:13 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:13.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:13 smithi064 ceph-mon[22742]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:13.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:13 smithi064 ceph-mon[22742]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:53:13.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:13 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:53:13.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:13 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:53:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:13 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:15.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:15 smithi064 ceph-mon[22742]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:15.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:15 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:15 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:53:15.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:15 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:15 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:15 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:15.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:15 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:15.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:15 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:15.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:15 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.463 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[21819]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:15.463 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.463 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:53:15.463 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[26536]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:15.464 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:15.465 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:15.465 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:15.465 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:15 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:16.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[26536]: Detected new or changed devices on smithi059 2024-04-26T15:53:16.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[26536]: Adjusting osd_memory_target on smithi059 to 16069M 2024-04-26T15:53:16.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/7548666' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "278729e0-965b-44fb-98c5-d5c87036f13d"}]: dispatch 2024-04-26T15:53:16.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "278729e0-965b-44fb-98c5-d5c87036f13d"}]: dispatch 2024-04-26T15:53:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "278729e0-965b-44fb-98c5-d5c87036f13d"}]': finished 2024-04-26T15:53:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[26536]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T15:53:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[21819]: Detected new or changed devices on smithi059 2024-04-26T15:53:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[21819]: Adjusting osd_memory_target on smithi059 to 16069M 2024-04-26T15:53:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/7548666' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "278729e0-965b-44fb-98c5-d5c87036f13d"}]: dispatch 2024-04-26T15:53:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "278729e0-965b-44fb-98c5-d5c87036f13d"}]: dispatch 2024-04-26T15:53:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "278729e0-965b-44fb-98c5-d5c87036f13d"}]': finished 2024-04-26T15:53:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[21819]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T15:53:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:16 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:16 smithi064 ceph-mon[22742]: Detected new or changed devices on smithi059 2024-04-26T15:53:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:16 smithi064 ceph-mon[22742]: Adjusting osd_memory_target on smithi059 to 16069M 2024-04-26T15:53:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:16 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/7548666' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "278729e0-965b-44fb-98c5-d5c87036f13d"}]: dispatch 2024-04-26T15:53:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:16 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "278729e0-965b-44fb-98c5-d5c87036f13d"}]: dispatch 2024-04-26T15:53:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:16 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "278729e0-965b-44fb-98c5-d5c87036f13d"}]': finished 2024-04-26T15:53:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:16 smithi064 ceph-mon[22742]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T15:53:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:16 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:17.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:17 smithi059 ceph-mon[21819]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:17.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:17 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2604212713' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:53:17.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:17 smithi059 ceph-mon[26536]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:17.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:17 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2604212713' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:53:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:17 smithi064 ceph-mon[22742]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:17 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2604212713' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:53:19.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:19 smithi059 ceph-mon[21819]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:19.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:19 smithi059 ceph-mon[26536]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:19.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:19 smithi064 ceph-mon[22742]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:21.537 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:21 smithi059 ceph-mon[21819]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:21.538 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:21 smithi059 ceph-mon[26536]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:21.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:21 smithi064 ceph-mon[22742]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:23.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:23 smithi059 ceph-mon[21819]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:23.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:23 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T15:53:23.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:23 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:23.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:23 smithi059 ceph-mon[21819]: Deploying daemon osd.1 on smithi059 2024-04-26T15:53:23.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:23 smithi059 ceph-mon[26536]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:23.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:23 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T15:53:23.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:23 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:23.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:23 smithi059 ceph-mon[26536]: Deploying daemon osd.1 on smithi059 2024-04-26T15:53:23.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:23 smithi064 ceph-mon[22742]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:23.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:23 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T15:53:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:23 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:23 smithi064 ceph-mon[22742]: Deploying daemon osd.1 on smithi059 2024-04-26T15:53:25.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:25 smithi059 ceph-mon[21819]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:25.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:25 smithi059 ceph-mon[26536]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:25.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:25 smithi064 ceph-mon[22742]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:27.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:27 smithi059 ceph-mon[21819]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:27.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:27 smithi059 ceph-mon[26536]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:27 smithi064 ceph-mon[22742]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:28.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:28 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:28.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:28 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:28 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:28 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:28 smithi059 ceph-mon[21819]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:28 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:28 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:28 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:28 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:28 smithi059 ceph-mon[26536]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:28.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:28 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:28.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:28 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:28.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:28 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:28 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:28 smithi064 ceph-mon[22742]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:30.990 INFO:teuthology.orchestra.run.smithi059.stdout:Created osd(s) 1 on host 'smithi059' 2024-04-26T15:53:31.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[21819]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[21819]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[26536]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[26536]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:31.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:31.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:31.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:31 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:31.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:31 smithi064 ceph-mon[22742]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:31.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:31 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:31.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:31 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:31.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:31 smithi064 ceph-mon[22742]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T15:53:31.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:31 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:31.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:31 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:31.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:31 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:31.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:31 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:31.686 DEBUG:teuthology.orchestra.run.smithi059:osd.1> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.1.service 2024-04-26T15:53:31.688 INFO:tasks.cephadm:Deploying osd.2 on smithi059 with /dev/vg_nvme/lv_2... 2024-04-26T15:53:31.688 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T15:53:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:32 smithi059 ceph-mon[21819]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T15:53:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:32 smithi059 ceph-mon[21819]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T15:53:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:32 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:32 smithi059 ceph-mon[21819]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:53:32.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:32 smithi059 ceph-mon[26536]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T15:53:32.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:32 smithi059 ceph-mon[26536]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T15:53:32.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:32 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:32.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:32 smithi059 ceph-mon[26536]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:53:32.334 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 15:53:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[32369]: 2024-04-26T15:53:32.123+0000 7f35c91cf700 -1 osd.1 0 waiting for initial osdmap 2024-04-26T15:53:32.334 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 15:53:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[32369]: 2024-04-26T15:53:32.127+0000 7f35c1b62700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:53:32.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:32 smithi064 ceph-mon[22742]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T15:53:32.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:32 smithi064 ceph-mon[22742]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T15:53:32.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:32 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:32.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:32 smithi064 ceph-mon[22742]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:53:33.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:33 smithi059 ceph-mon[21819]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:33.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:33 smithi059 ceph-mon[21819]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:53:33.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:33 smithi059 ceph-mon[21819]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T15:53:33.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:33.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:33 smithi059 ceph-mon[26536]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:33 smithi059 ceph-mon[26536]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:53:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:33 smithi059 ceph-mon[26536]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T15:53:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:33 smithi064 ceph-mon[22742]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:33 smithi064 ceph-mon[22742]: from='osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:53:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:33 smithi064 ceph-mon[22742]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T15:53:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:34.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:34 smithi064 ceph-mon[22742]: purged_snaps scrub starts 2024-04-26T15:53:34.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:34 smithi064 ceph-mon[22742]: purged_snaps scrub ok 2024-04-26T15:53:34.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:34 smithi064 ceph-mon[22742]: osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573] boot 2024-04-26T15:53:34.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:34 smithi064 ceph-mon[22742]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T15:53:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:34 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:34.462 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:34 smithi059 ceph-mon[26536]: purged_snaps scrub starts 2024-04-26T15:53:34.463 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:34 smithi059 ceph-mon[26536]: purged_snaps scrub ok 2024-04-26T15:53:34.463 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:34 smithi059 ceph-mon[26536]: osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573] boot 2024-04-26T15:53:34.463 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:34 smithi059 ceph-mon[26536]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T15:53:34.463 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:34 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:34.463 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:34 smithi059 ceph-mon[21819]: purged_snaps scrub starts 2024-04-26T15:53:34.463 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:34 smithi059 ceph-mon[21819]: purged_snaps scrub ok 2024-04-26T15:53:34.463 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:34 smithi059 ceph-mon[21819]: osd.1 [v2:172.21.15.59:6810/448553573,v1:172.21.15.59:6811/448553573] boot 2024-04-26T15:53:34.463 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:34 smithi059 ceph-mon[21819]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T15:53:34.463 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:34 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:53:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:35 smithi064 ceph-mon[22742]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:35 smithi064 ceph-mon[22742]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T15:53:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:35 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:35 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:53:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:35 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:53:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:35 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:35 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:35.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[26536]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:35.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[26536]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[21819]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[21819]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:53:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:35.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:35 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:35.675 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T15:53:35.675 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T15:53:35.675 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10+0 records in 2024-04-26T15:53:35.675 INFO:teuthology.orchestra.run.smithi059.stderr:10+0 records out 2024-04-26T15:53:35.675 INFO:teuthology.orchestra.run.smithi059.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110901 s, 946 MB/s 2024-04-26T15:53:35.676 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping successful for: 2024-04-26T15:53:36.306 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:36 smithi059 ceph-mon[26536]: Detected new or changed devices on smithi059 2024-04-26T15:53:36.306 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:36 smithi059 ceph-mon[26536]: Adjusting osd_memory_target on smithi059 to 8034M 2024-04-26T15:53:36.306 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:36 smithi059 ceph-mon[21819]: Detected new or changed devices on smithi059 2024-04-26T15:53:36.306 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:36 smithi059 ceph-mon[21819]: Adjusting osd_memory_target on smithi059 to 8034M 2024-04-26T15:53:36.336 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi059:vg_nvme/lv_2 2024-04-26T15:53:36.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:36 smithi064 ceph-mon[22742]: Detected new or changed devices on smithi059 2024-04-26T15:53:36.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:36 smithi064 ceph-mon[22742]: Adjusting osd_memory_target on smithi059 to 8034M 2024-04-26T15:53:37.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:37 smithi059 ceph-mon[21819]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:37.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:37 smithi059 ceph-mon[26536]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:37.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:37 smithi064 ceph-mon[22742]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:39.298 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:39 smithi059 ceph-mon[21819]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:39.298 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:39 smithi059 ceph-mon[21819]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:53:39.298 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:39 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:53:39.298 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:39 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:53:39.298 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:39 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:39.298 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:39 smithi059 ceph-mon[26536]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:39.298 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:39 smithi059 ceph-mon[26536]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:53:39.298 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:39 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:53:39.299 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:39 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:53:39.299 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:39 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:39 smithi064 ceph-mon[22742]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:39 smithi064 ceph-mon[22742]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:53:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:53:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:53:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:39 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:41.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:41 smithi064 ceph-mon[22742]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:41.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:41 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1550103214' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfd237b8-d7ea-4c31-a514-5f499db07cea"}]: dispatch 2024-04-26T15:53:41.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:41 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfd237b8-d7ea-4c31-a514-5f499db07cea"}]: dispatch 2024-04-26T15:53:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:41 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfd237b8-d7ea-4c31-a514-5f499db07cea"}]': finished 2024-04-26T15:53:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:41 smithi064 ceph-mon[22742]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T15:53:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:41 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[21819]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1550103214' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfd237b8-d7ea-4c31-a514-5f499db07cea"}]: dispatch 2024-04-26T15:53:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfd237b8-d7ea-4c31-a514-5f499db07cea"}]: dispatch 2024-04-26T15:53:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfd237b8-d7ea-4c31-a514-5f499db07cea"}]': finished 2024-04-26T15:53:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[21819]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T15:53:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[26536]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1550103214' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfd237b8-d7ea-4c31-a514-5f499db07cea"}]: dispatch 2024-04-26T15:53:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfd237b8-d7ea-4c31-a514-5f499db07cea"}]: dispatch 2024-04-26T15:53:41.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfd237b8-d7ea-4c31-a514-5f499db07cea"}]': finished 2024-04-26T15:53:41.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[26536]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T15:53:41.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:41 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:42.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:42 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2308910990' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:53:42.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:42 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2308910990' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:53:42.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:42 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2308910990' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:53:43.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:43 smithi064 ceph-mon[22742]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:43.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:43 smithi059 ceph-mon[21819]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:43.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:43 smithi059 ceph-mon[26536]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:45.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:45 smithi059 ceph-mon[26536]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:45.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:45 smithi059 ceph-mon[21819]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:45.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:45 smithi064 ceph-mon[22742]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:47.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:47 smithi059 ceph-mon[26536]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:47.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T15:53:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:47.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:47 smithi059 ceph-mon[21819]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:47.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T15:53:47.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:47.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:47 smithi064 ceph-mon[22742]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:47.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T15:53:47.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:48.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:48 smithi064 ceph-mon[22742]: Deploying daemon osd.2 on smithi059 2024-04-26T15:53:48.483 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:48 smithi059 ceph-mon[21819]: Deploying daemon osd.2 on smithi059 2024-04-26T15:53:48.483 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:48 smithi059 ceph-mon[26536]: Deploying daemon osd.2 on smithi059 2024-04-26T15:53:49.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:49 smithi064 ceph-mon[22742]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:49.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:49 smithi059 ceph-mon[21819]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:49.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:49 smithi059 ceph-mon[26536]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:51.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:51 smithi064 ceph-mon[22742]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:51.438 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:51 smithi059 ceph-mon[21819]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:51.439 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:51 smithi059 ceph-mon[26536]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:52.278 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:52 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:53:52.279 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:52 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:53:52.279 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:53:52.279 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:53:52.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:52 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:53:52.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:52 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:53:53.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:53 smithi059 ceph-mon[21819]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:53.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:53 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:53.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:53 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:53.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:53 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:53.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:53 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:53 smithi059 ceph-mon[26536]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:53 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:53 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:53 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:53 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:53.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:53 smithi064 ceph-mon[22742]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:53.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:53 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:53.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:53 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:53:53.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:53 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:53:53.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:53 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:53:55.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:55 smithi059 ceph-mon[21819]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:55.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:55 smithi059 ceph-mon[26536]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:55.388 INFO:teuthology.orchestra.run.smithi059.stdout:Created osd(s) 2 on host 'smithi059' 2024-04-26T15:53:55.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:55 smithi064 ceph-mon[22742]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:55.972 DEBUG:teuthology.orchestra.run.smithi059:osd.2> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.2.service 2024-04-26T15:53:55.974 INFO:tasks.cephadm:Deploying osd.3 on smithi059 with /dev/vg_nvme/lv_1... 2024-04-26T15:53:55.974 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T15:53:56.473 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:56 smithi059 ceph-mon[26536]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:53:56.473 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:56 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:56.473 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:56 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:56.473 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:56 smithi059 ceph-mon[21819]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:53:56.474 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:56 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:56.474 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:56 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:56.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:56 smithi064 ceph-mon[22742]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T15:53:56.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:56 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:56.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:56 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:53:57.368 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-mon[26536]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:57.368 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-mon[26536]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T15:53:57.368 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-mon[26536]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T15:53:57.368 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:57.368 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-mon[26536]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:53:57.369 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-mon[21819]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:57.369 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-mon[21819]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T15:53:57.369 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-mon[21819]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T15:53:57.369 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:57.369 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-mon[21819]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:53:57.369 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[35313]: 2024-04-26T15:53:57.271+0000 7fda64a7b700 -1 osd.2 0 waiting for initial osdmap 2024-04-26T15:53:57.369 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 15:53:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[35313]: 2024-04-26T15:53:57.274+0000 7fda5ec11700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:53:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:57 smithi064 ceph-mon[22742]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:57 smithi064 ceph-mon[22742]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T15:53:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:57 smithi064 ceph-mon[22742]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T15:53:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:57 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:57 smithi064 ceph-mon[22742]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:53:58.105 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T15:53:58.105 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T15:53:58.105 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10+0 records in 2024-04-26T15:53:58.105 INFO:teuthology.orchestra.run.smithi059.stderr:10+0 records out 2024-04-26T15:53:58.105 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129399 s, 810 MB/s 2024-04-26T15:53:58.106 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping successful for: 2024-04-26T15:53:58.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:58 smithi059 ceph-mon[21819]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:53:58.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:58 smithi059 ceph-mon[21819]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T15:53:58.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:58 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:58.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:58 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:58.524 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:58 smithi059 ceph-mon[26536]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:53:58.524 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:58 smithi059 ceph-mon[26536]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T15:53:58.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:58 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:58.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:58 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:58.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:58 smithi064 ceph-mon[22742]: from='osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:53:58.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:58 smithi064 ceph-mon[22742]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T15:53:58.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:58 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:58.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:58 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:59.425 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[21819]: purged_snaps scrub starts 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[21819]: purged_snaps scrub ok 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[21819]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[21819]: osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792] boot 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[21819]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[26536]: purged_snaps scrub starts 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[26536]: purged_snaps scrub ok 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[26536]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:59.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[26536]: osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792] boot 2024-04-26T15:53:59.427 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[26536]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T15:53:59.427 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:53:59 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:59 smithi064 ceph-mon[22742]: purged_snaps scrub starts 2024-04-26T15:53:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:59 smithi064 ceph-mon[22742]: purged_snaps scrub ok 2024-04-26T15:53:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:59 smithi064 ceph-mon[22742]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-26T15:53:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:59 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:59.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:59 smithi064 ceph-mon[22742]: osd.2 [v2:172.21.15.59:6818/2282009792,v1:172.21.15.59:6819/2282009792] boot 2024-04-26T15:53:59.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:59 smithi064 ceph-mon[22742]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T15:53:59.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:53:59 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:53:59.911 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi059:vg_nvme/lv_1 2024-04-26T15:54:00.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:00 smithi059 ceph-mon[21819]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T15:54:00.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:00 smithi059 ceph-mon[26536]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T15:54:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:00 smithi064 ceph-mon[22742]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T15:54:01.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:01.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:01.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:01.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.587 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:01.587 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:02.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:02 smithi064 ceph-mon[22742]: Detected new or changed devices on smithi059 2024-04-26T15:54:02.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:02 smithi064 ceph-mon[22742]: Adjusting osd_memory_target on smithi059 to 5356M 2024-04-26T15:54:02.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:02 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T15:54:02.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:02 smithi064 ceph-mon[22742]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T15:54:02.690 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:02 smithi059 ceph-mon[21819]: Detected new or changed devices on smithi059 2024-04-26T15:54:02.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:02 smithi059 ceph-mon[21819]: Adjusting osd_memory_target on smithi059 to 5356M 2024-04-26T15:54:02.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:02 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T15:54:02.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:02 smithi059 ceph-mon[21819]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T15:54:02.691 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:02 smithi059 ceph-mon[26536]: Detected new or changed devices on smithi059 2024-04-26T15:54:02.691 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:02 smithi059 ceph-mon[26536]: Adjusting osd_memory_target on smithi059 to 5356M 2024-04-26T15:54:02.691 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:02 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T15:54:02.691 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:02 smithi059 ceph-mon[26536]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T15:54:03.002 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 15:54:02 smithi059 sudo[36712]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:54:03.002 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 15:54:02 smithi059 sudo[36712]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:03.002 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 15:54:02 smithi059 sudo[36712]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:03.332 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 15:54:03 smithi059 sudo[36771]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:54:03.332 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 15:54:03 smithi059 sudo[36771]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:03.332 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 15:54:03 smithi059 sudo[36771]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:03.630 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[26536]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[26536]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[21819]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[21819]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T15:54:03.631 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:03 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:03.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:03 smithi064 ceph-mon[22742]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:03.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:03 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:54:03.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:03 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:54:03.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:03 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:03.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:03 smithi064 ceph-mon[22742]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T15:54:03.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:03 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:03.963 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 15:54:03 smithi059 sudo[36778]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:54:03.963 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 15:54:03 smithi059 sudo[36778]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:03.963 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 15:54:03 smithi059 sudo[36778]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:04.260 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 15:54:04 smithi059 sudo[36804]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:54:04.260 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 15:54:04 smithi059 sudo[36804]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:04.260 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 15:54:04 smithi059 sudo[36804]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:04.535 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:04 smithi059 ceph-mon[21819]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:54:04.535 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:04 smithi059 ceph-mon[26536]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:54:04.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:04 smithi064 ceph-mon[22742]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:54:04.832 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 15:54:04 smithi059 sudo[36819]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:54:04.832 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 15:54:04 smithi059 sudo[36819]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:04.832 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 15:54:04 smithi059 sudo[36819]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:05.332 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 15:54:04 smithi059 sudo[36851]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:54:05.332 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 15:54:04 smithi059 sudo[36851]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:05.332 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 15:54:04 smithi059 sudo[36851]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:05.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 sudo[36883]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:54:05.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 sudo[36883]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:05.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 sudo[36883]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:06.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:05 smithi064 ceph-mon[22742]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:06.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:05 smithi064 ceph-mon[22742]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T15:54:06.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:05 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1775444352' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58994c22-3506-45b6-a449-81b31271ed94"}]: dispatch 2024-04-26T15:54:06.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:05 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58994c22-3506-45b6-a449-81b31271ed94"}]: dispatch 2024-04-26T15:54:06.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:05 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58994c22-3506-45b6-a449-81b31271ed94"}]': finished 2024-04-26T15:54:06.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:05 smithi064 ceph-mon[22742]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T15:54:06.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:05 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:06.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[26536]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:06.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[26536]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T15:54:06.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1775444352' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58994c22-3506-45b6-a449-81b31271ed94"}]: dispatch 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58994c22-3506-45b6-a449-81b31271ed94"}]: dispatch 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58994c22-3506-45b6-a449-81b31271ed94"}]': finished 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[26536]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 sudo[36936]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 sudo[36936]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 sudo[36936]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[21819]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[21819]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1775444352' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58994c22-3506-45b6-a449-81b31271ed94"}]: dispatch 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58994c22-3506-45b6-a449-81b31271ed94"}]: dispatch 2024-04-26T15:54:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58994c22-3506-45b6-a449-81b31271ed94"}]': finished 2024-04-26T15:54:06.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[21819]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T15:54:06.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:05 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:06.794 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:06 smithi059 sudo[37310]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:54:06.794 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:06 smithi059 sudo[37310]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:06.794 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:06 smithi059 sudo[37310]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:07.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:06 smithi059 sudo[37314]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:54:07.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:06 smithi059 sudo[37314]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:07.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:06 smithi059 sudo[37314]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:07.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 sudo[24260]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T15:54:07.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 sudo[24260]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:07.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 sudo[24260]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:08.067 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:54:08.067 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1424280103' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:54:08.067 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:08.067 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:54:08.067 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:54:08.068 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1424280103' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:54:08.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:54:08.070 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:54:08.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 sudo[24263]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T15:54:08.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 sudo[24263]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 sudo[24263]: pam_unix(sudo:session): session closed for user root 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1424280103' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:54:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:54:08.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:54:08.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:54:08.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:54:08.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:54:09.002 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:08 smithi059 ceph-mon[26536]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:54:09.002 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:08 smithi059 ceph-mon[26536]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:54:09.002 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:08 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:09.002 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:08 smithi059 ceph-mon[26536]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:09.003 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:08 smithi059 ceph-mon[21819]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:54:09.003 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:08 smithi059 ceph-mon[21819]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:54:09.003 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:08 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:09.003 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:08 smithi059 ceph-mon[21819]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:09.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:08 smithi064 ceph-mon[22742]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T15:54:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:08 smithi064 ceph-mon[22742]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T15:54:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:08 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:08 smithi064 ceph-mon[22742]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:11.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:10 smithi059 ceph-mon[21819]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.6 KiB/s rd, 158 KiB/s wr, 11 op/s 2024-04-26T15:54:11.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:10 smithi059 ceph-mon[26536]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.6 KiB/s rd, 158 KiB/s wr, 11 op/s 2024-04-26T15:54:11.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:10 smithi064 ceph-mon[22742]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.6 KiB/s rd, 158 KiB/s wr, 11 op/s 2024-04-26T15:54:11.933 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:11 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T15:54:11.933 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:11 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:11.933 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:11 smithi059 ceph-mon[26536]: Deploying daemon osd.3 on smithi059 2024-04-26T15:54:11.934 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T15:54:11.934 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:11.934 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:11 smithi059 ceph-mon[21819]: Deploying daemon osd.3 on smithi059 2024-04-26T15:54:12.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:11 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T15:54:12.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:11 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:12.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:11 smithi064 ceph-mon[22742]: Deploying daemon osd.3 on smithi059 2024-04-26T15:54:13.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:13 smithi059 ceph-mon[26536]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 113 KiB/s wr, 8 op/s 2024-04-26T15:54:13.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:13 smithi059 ceph-mon[21819]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 113 KiB/s wr, 8 op/s 2024-04-26T15:54:13.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:13 smithi064 ceph-mon[22742]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 113 KiB/s wr, 8 op/s 2024-04-26T15:54:15.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:15 smithi059 ceph-mon[21819]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.0 KiB/s rd, 87 KiB/s wr, 6 op/s 2024-04-26T15:54:15.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:15 smithi059 ceph-mon[26536]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.0 KiB/s rd, 87 KiB/s wr, 6 op/s 2024-04-26T15:54:15.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:15 smithi064 ceph-mon[22742]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.0 KiB/s rd, 87 KiB/s wr, 6 op/s 2024-04-26T15:54:17.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:17 smithi059 ceph-mon[21819]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 72 KiB/s wr, 5 op/s 2024-04-26T15:54:17.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:17 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:17.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:17 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:17.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:17 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:17.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:17 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:17.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:17 smithi059 ceph-mon[26536]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 72 KiB/s wr, 5 op/s 2024-04-26T15:54:17.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:17 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:17.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:17 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:17.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:17 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:17.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:17 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:17 smithi064 ceph-mon[22742]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 72 KiB/s wr, 5 op/s 2024-04-26T15:54:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:17 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:17 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:17 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:17 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:19.499 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:19 smithi059 ceph-mon[21819]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T15:54:19.500 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:19 smithi059 ceph-mon[26536]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T15:54:19.533 INFO:teuthology.orchestra.run.smithi059.stdout:Created osd(s) 3 on host 'smithi059' 2024-04-26T15:54:19.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:19 smithi064 ceph-mon[22742]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T15:54:20.137 DEBUG:teuthology.orchestra.run.smithi059:osd.3> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.3.service 2024-04-26T15:54:20.140 INFO:tasks.cephadm:Deploying osd.4 on smithi064 with /dev/vg_nvme/lv_4... 2024-04-26T15:54:20.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T15:54:20.413 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:20 smithi059 ceph-mon[26536]: from='osd.3 [v2:172.21.15.59:6826/4167347038,v1:172.21.15.59:6827/4167347038]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T15:54:20.413 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:20 smithi059 ceph-mon[26536]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T15:54:20.413 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:20 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:20.414 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:20 smithi059 ceph-mon[21819]: from='osd.3 [v2:172.21.15.59:6826/4167347038,v1:172.21.15.59:6827/4167347038]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T15:54:20.414 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:20 smithi059 ceph-mon[21819]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T15:54:20.414 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:20 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:20.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:20 smithi064 ceph-mon[22742]: from='osd.3 [v2:172.21.15.59:6826/4167347038,v1:172.21.15.59:6827/4167347038]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T15:54:20.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:20 smithi064 ceph-mon[22742]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T15:54:20.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:20 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:21.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[21819]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T15:54:21.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[21819]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T15:54:21.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[21819]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T15:54:21.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:21.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[21819]: from='osd.3 [v2:172.21.15.59:6826/4167347038,v1:172.21.15.59:6827/4167347038]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:54:21.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[21819]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:54:21.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:21.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[26536]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T15:54:21.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[26536]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T15:54:21.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[26536]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T15:54:21.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:21.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[26536]: from='osd.3 [v2:172.21.15.59:6826/4167347038,v1:172.21.15.59:6827/4167347038]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:54:21.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[26536]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:54:21.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:21.334 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[38311]: 2024-04-26T15:54:21.288+0000 7f4b95dd7700 -1 osd.3 0 waiting for initial osdmap 2024-04-26T15:54:21.334 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 15:54:21 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[38311]: 2024-04-26T15:54:21.292+0000 7f4b92772700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:54:21.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:21 smithi064 ceph-mon[22742]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T15:54:21.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:21 smithi064 ceph-mon[22742]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T15:54:21.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:21 smithi064 ceph-mon[22742]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T15:54:21.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:21 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:21.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:21 smithi064 ceph-mon[22742]: from='osd.3 [v2:172.21.15.59:6826/4167347038,v1:172.21.15.59:6827/4167347038]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:54:21.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:21 smithi064 ceph-mon[22742]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T15:54:21.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:21 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:22.240 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T15:54:22.240 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T15:54:22.241 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2024-04-26T15:54:22.241 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2024-04-26T15:54:22.241 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114016 s, 920 MB/s 2024-04-26T15:54:22.241 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2024-04-26T15:54:22.519 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:22 smithi064 ceph-mon[22742]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:54:22.520 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:22 smithi064 ceph-mon[22742]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T15:54:22.520 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:22 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:22.520 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:22 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:22.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:22 smithi059 ceph-mon[21819]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:54:22.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:22 smithi059 ceph-mon[21819]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T15:54:22.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:22 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:22.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:22 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:22 smithi059 ceph-mon[26536]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2024-04-26T15:54:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:22 smithi059 ceph-mon[26536]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T15:54:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:22 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:22 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:22.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi064:vg_nvme/lv_4 2024-04-26T15:54:23.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[21819]: purged_snaps scrub starts 2024-04-26T15:54:23.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[21819]: purged_snaps scrub ok 2024-04-26T15:54:23.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[21819]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:23.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[21819]: osd.3 [v2:172.21.15.59:6826/4167347038,v1:172.21.15.59:6827/4167347038] boot 2024-04-26T15:54:23.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[21819]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T15:54:23.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:23 smithi064 ceph-mon[22742]: purged_snaps scrub starts 2024-04-26T15:54:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:23 smithi064 ceph-mon[22742]: purged_snaps scrub ok 2024-04-26T15:54:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:23 smithi064 ceph-mon[22742]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:23 smithi064 ceph-mon[22742]: osd.3 [v2:172.21.15.59:6826/4167347038,v1:172.21.15.59:6827/4167347038] boot 2024-04-26T15:54:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:23 smithi064 ceph-mon[22742]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T15:54:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:23 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:23.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:23.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[26536]: purged_snaps scrub starts 2024-04-26T15:54:23.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[26536]: purged_snaps scrub ok 2024-04-26T15:54:23.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[26536]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-26T15:54:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[26536]: osd.3 [v2:172.21.15.59:6826/4167347038,v1:172.21.15.59:6827/4167347038] boot 2024-04-26T15:54:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[26536]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T15:54:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:23 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:54:24.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:25.378 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:25 smithi064 ceph-mon[22742]: Detected new or changed devices on smithi059 2024-04-26T15:54:25.378 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:25 smithi064 ceph-mon[22742]: Adjusting osd_memory_target on smithi059 to 4017M 2024-04-26T15:54:25.378 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:25 smithi064 ceph-mon[22742]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:25.378 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:25 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:54:25.378 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:25 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:54:25.378 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:25 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:25.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[21819]: Detected new or changed devices on smithi059 2024-04-26T15:54:25.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[21819]: Adjusting osd_memory_target on smithi059 to 4017M 2024-04-26T15:54:25.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[21819]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:25.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:54:25.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:54:25.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:25.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[26536]: Detected new or changed devices on smithi059 2024-04-26T15:54:25.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[26536]: Adjusting osd_memory_target on smithi059 to 4017M 2024-04-26T15:54:25.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[26536]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:25.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:54:25.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:54:25.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:25 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:26.401 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:26 smithi064 ceph-mon[22742]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:54:26.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:26 smithi059 ceph-mon[21819]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:54:26.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:26 smithi059 ceph-mon[26536]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:54:27.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[21819]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/125054966' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "193191de-111e-4026-a251-a4a1f9a7a8cf"}]: dispatch 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "193191de-111e-4026-a251-a4a1f9a7a8cf"}]: dispatch 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "193191de-111e-4026-a251-a4a1f9a7a8cf"}]': finished 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[21819]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[26536]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/125054966' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "193191de-111e-4026-a251-a4a1f9a7a8cf"}]: dispatch 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "193191de-111e-4026-a251-a4a1f9a7a8cf"}]: dispatch 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "193191de-111e-4026-a251-a4a1f9a7a8cf"}]': finished 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[26536]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T15:54:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:27 smithi064 ceph-mon[22742]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:27 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/125054966' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "193191de-111e-4026-a251-a4a1f9a7a8cf"}]: dispatch 2024-04-26T15:54:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:27 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "193191de-111e-4026-a251-a4a1f9a7a8cf"}]: dispatch 2024-04-26T15:54:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:27 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "193191de-111e-4026-a251-a4a1f9a7a8cf"}]': finished 2024-04-26T15:54:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:27 smithi064 ceph-mon[22742]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T15:54:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:27 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:28.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:28 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/385328891' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:54:28.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:28 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/385328891' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:54:28.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:28 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/385328891' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:54:29.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:29 smithi059 ceph-mon[21819]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:29.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:29 smithi059 ceph-mon[26536]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:29.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:29 smithi064 ceph-mon[22742]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:31.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:31 smithi059 ceph-mon[21819]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:31.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:31 smithi059 ceph-mon[26536]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:31.603 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:31 smithi064 ceph-mon[22742]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:33.506 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:33 smithi064 ceph-mon[22742]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:33.506 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T15:54:33.506 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:33.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:33 smithi059 ceph-mon[21819]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T15:54:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:33 smithi059 ceph-mon[26536]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T15:54:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:34.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:34 smithi064 ceph-mon[22742]: Deploying daemon osd.4 on smithi064 2024-04-26T15:54:34.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:34 smithi059 ceph-mon[21819]: Deploying daemon osd.4 on smithi064 2024-04-26T15:54:34.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:34 smithi059 ceph-mon[26536]: Deploying daemon osd.4 on smithi064 2024-04-26T15:54:35.501 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:35 smithi064 ceph-mon[22742]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:35.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:35 smithi059 ceph-mon[21819]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:35.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:35 smithi059 ceph-mon[26536]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:37.426 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:37 smithi064 ceph-mon[22742]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:37.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:37 smithi059 ceph-mon[21819]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:37.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:37 smithi059 ceph-mon[26536]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:39.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:38 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:39.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:38 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:39.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:38 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:39.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:38 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:39.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:38 smithi059 ceph-mon[21819]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:38 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:38 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:38 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:38 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:38 smithi059 ceph-mon[26536]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:39.096 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:38 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:39.096 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:38 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:39.096 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:38 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:39.096 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:38 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:39.096 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:38 smithi064 ceph-mon[22742]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:40.754 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 4 on host 'smithi064' 2024-04-26T15:54:41.022 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:40 smithi064 ceph-mon[22742]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:41.022 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:40 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:41.023 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:40 smithi064 ceph-mon[22742]: from='osd.4 [v2:172.21.15.64:6800/304831184,v1:172.21.15.64:6801/304831184]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:54:41.023 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:40 smithi064 ceph-mon[22742]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:54:41.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:40 smithi059 ceph-mon[21819]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:41.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:40 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:41.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:40 smithi059 ceph-mon[21819]: from='osd.4 [v2:172.21.15.64:6800/304831184,v1:172.21.15.64:6801/304831184]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:54:41.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:40 smithi059 ceph-mon[21819]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:54:41.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:40 smithi059 ceph-mon[26536]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:41.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:40 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:41.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:40 smithi059 ceph-mon[26536]: from='osd.4 [v2:172.21.15.64:6800/304831184,v1:172.21.15.64:6801/304831184]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:54:41.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:40 smithi059 ceph-mon[26536]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T15:54:41.414 DEBUG:teuthology.orchestra.run.smithi064:osd.4> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.4.service 2024-04-26T15:54:41.417 INFO:tasks.cephadm:Deploying osd.5 on smithi064 with /dev/vg_nvme/lv_3... 2024-04-26T15:54:41.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T15:54:42.312 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:42 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:42.312 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:42 smithi064 ceph-mon[22742]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T15:54:42.312 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:42 smithi064 ceph-mon[22742]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T15:54:42.312 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:42 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:42.312 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:42 smithi064 ceph-mon[22742]: from='osd.4 [v2:172.21.15.64:6800/304831184,v1:172.21.15.64:6801/304831184]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:54:42.312 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:42 smithi064 ceph-mon[22742]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:54:42.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:42.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[21819]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T15:54:42.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[21819]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T15:54:42.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:42.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[21819]: from='osd.4 [v2:172.21.15.64:6800/304831184,v1:172.21.15.64:6801/304831184]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:54:42.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[21819]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:54:42.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:42.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[26536]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T15:54:42.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[26536]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T15:54:42.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:42.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[26536]: from='osd.4 [v2:172.21.15.64:6800/304831184,v1:172.21.15.64:6801/304831184]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:54:42.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:42 smithi059 ceph-mon[26536]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:54:42.937 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 15:54:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[26173]: 2024-04-26T15:54:42.765+0000 7f4849ed6700 -1 osd.4 0 waiting for initial osdmap 2024-04-26T15:54:42.937 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 15:54:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[26173]: 2024-04-26T15:54:42.770+0000 7f484306a700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:54:43.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:43 smithi059 ceph-mon[21819]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:43.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:43 smithi059 ceph-mon[21819]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:54:43.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:43 smithi059 ceph-mon[21819]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T15:54:43.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:43 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:43.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:43 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:43.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:43 smithi059 ceph-mon[26536]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:43.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:43 smithi059 ceph-mon[26536]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:54:43.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:43 smithi059 ceph-mon[26536]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T15:54:43.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:43 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:43.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:43 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:43.365 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:43 smithi064 ceph-mon[22742]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-26T15:54:43.365 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:43 smithi064 ceph-mon[22742]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:54:43.365 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:43 smithi064 ceph-mon[22742]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T15:54:43.365 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:43 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:43.365 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:43 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:44.366 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T15:54:44.367 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T15:54:44.367 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2024-04-26T15:54:44.367 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2024-04-26T15:54:44.367 INFO:teuthology.orchestra.run.smithi064.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.012306 s, 852 MB/s 2024-04-26T15:54:44.367 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2024-04-26T15:54:45.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[21819]: purged_snaps scrub starts 2024-04-26T15:54:45.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[21819]: purged_snaps scrub ok 2024-04-26T15:54:45.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[21819]: osd.4 [v2:172.21.15.64:6800/304831184,v1:172.21.15.64:6801/304831184] boot 2024-04-26T15:54:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[21819]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T15:54:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[21819]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:54:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[26536]: purged_snaps scrub starts 2024-04-26T15:54:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[26536]: purged_snaps scrub ok 2024-04-26T15:54:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[26536]: osd.4 [v2:172.21.15.64:6800/304831184,v1:172.21.15.64:6801/304831184] boot 2024-04-26T15:54:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[26536]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T15:54:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:44 smithi059 ceph-mon[26536]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:54:45.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:44 smithi064 ceph-mon[22742]: purged_snaps scrub starts 2024-04-26T15:54:45.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:44 smithi064 ceph-mon[22742]: purged_snaps scrub ok 2024-04-26T15:54:45.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:44 smithi064 ceph-mon[22742]: osd.4 [v2:172.21.15.64:6800/304831184,v1:172.21.15.64:6801/304831184] boot 2024-04-26T15:54:45.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:44 smithi064 ceph-mon[22742]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T15:54:45.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:44 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:54:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:44 smithi064 ceph-mon[22742]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:54:45.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi064:vg_nvme/lv_3 2024-04-26T15:54:46.056 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:45 smithi064 ceph-mon[22742]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T15:54:46.057 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.057 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:46.057 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.057 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.057 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:46.057 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:46.057 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:46.057 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:45 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[21819]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T15:54:46.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:46.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[26536]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:54:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:54:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:45 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:54:47.028 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:46 smithi064 ceph-mon[22742]: Detected new or changed devices on smithi064 2024-04-26T15:54:47.029 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:46 smithi064 ceph-mon[22742]: Adjusting osd_memory_target on smithi064 to 17088M 2024-04-26T15:54:47.029 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:46 smithi064 ceph-mon[22742]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T15:54:47.029 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:46 smithi064 ceph-mon[22742]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:54:47.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:46 smithi059 ceph-mon[26536]: Detected new or changed devices on smithi064 2024-04-26T15:54:47.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:46 smithi059 ceph-mon[26536]: Adjusting osd_memory_target on smithi064 to 17088M 2024-04-26T15:54:47.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:46 smithi059 ceph-mon[26536]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T15:54:47.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:46 smithi059 ceph-mon[26536]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:54:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:46 smithi059 ceph-mon[21819]: Detected new or changed devices on smithi064 2024-04-26T15:54:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:46 smithi059 ceph-mon[21819]: Adjusting osd_memory_target on smithi064 to 17088M 2024-04-26T15:54:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:46 smithi059 ceph-mon[21819]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T15:54:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:46 smithi059 ceph-mon[21819]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:54:48.039 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:47 smithi064 ceph-mon[22742]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T15:54:48.040 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:47 smithi064 ceph-mon[22742]: from='client.34235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:54:48.040 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:54:48.040 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:54:48.040 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:48.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:47 smithi059 ceph-mon[26536]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T15:54:48.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:47 smithi059 ceph-mon[26536]: from='client.34235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:54:48.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:54:48.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:54:48.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:47 smithi059 ceph-mon[21819]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T15:54:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:47 smithi059 ceph-mon[21819]: from='client.34235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:54:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:54:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:54:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:48.891 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:48 smithi064 ceph-mon[22742]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T15:54:48.891 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:48 smithi064 ceph-mon[22742]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:54:49.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:48 smithi059 ceph-mon[26536]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T15:54:49.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:48 smithi059 ceph-mon[26536]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:54:49.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:48 smithi059 ceph-mon[21819]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T15:54:49.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:48 smithi059 ceph-mon[21819]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:54:50.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:49 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/2768552237' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "601697da-23df-4269-a707-eb9435930d2f"}]: dispatch 2024-04-26T15:54:50.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:49 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "601697da-23df-4269-a707-eb9435930d2f"}]: dispatch 2024-04-26T15:54:50.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:49 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "601697da-23df-4269-a707-eb9435930d2f"}]': finished 2024-04-26T15:54:50.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:49 smithi059 ceph-mon[26536]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T15:54:50.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:49 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:54:50.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:49 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/2768552237' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "601697da-23df-4269-a707-eb9435930d2f"}]: dispatch 2024-04-26T15:54:50.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:49 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "601697da-23df-4269-a707-eb9435930d2f"}]: dispatch 2024-04-26T15:54:50.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:49 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "601697da-23df-4269-a707-eb9435930d2f"}]': finished 2024-04-26T15:54:50.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:49 smithi059 ceph-mon[21819]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T15:54:50.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:49 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:54:50.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:49 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/2768552237' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "601697da-23df-4269-a707-eb9435930d2f"}]: dispatch 2024-04-26T15:54:50.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:49 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "601697da-23df-4269-a707-eb9435930d2f"}]: dispatch 2024-04-26T15:54:50.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:49 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "601697da-23df-4269-a707-eb9435930d2f"}]': finished 2024-04-26T15:54:50.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:49 smithi064 ceph-mon[22742]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T15:54:50.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:49 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:54:51.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:50 smithi059 ceph-mon[26536]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-04-26T15:54:51.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:50 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/1551274948' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:54:51.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:50 smithi059 ceph-mon[21819]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-04-26T15:54:51.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:50 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/1551274948' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:54:51.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:50 smithi064 ceph-mon[22742]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-04-26T15:54:51.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:50 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/1551274948' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:54:52.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:54:52.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:54:52.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:54:52.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:54:52.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:54:52.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:54:53.085 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:53 smithi064 ceph-mon[22742]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:54:53.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:53 smithi059 ceph-mon[26536]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:54:53.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:53 smithi059 ceph-mon[21819]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:54:55.180 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:55 smithi064 ceph-mon[22742]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-04-26T15:54:55.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:55 smithi059 ceph-mon[26536]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-04-26T15:54:55.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:55 smithi059 ceph-mon[21819]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-04-26T15:54:56.322 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:56 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T15:54:56.322 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:56 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:56.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:56 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T15:54:56.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:56 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:56.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:56 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T15:54:56.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:56 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:54:57.281 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:57 smithi064 ceph-mon[22742]: Deploying daemon osd.5 on smithi064 2024-04-26T15:54:57.281 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:57 smithi064 ceph-mon[22742]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-26T15:54:57.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:57 smithi059 ceph-mon[26536]: Deploying daemon osd.5 on smithi064 2024-04-26T15:54:57.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:57 smithi059 ceph-mon[26536]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-26T15:54:57.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:57 smithi059 ceph-mon[21819]: Deploying daemon osd.5 on smithi064 2024-04-26T15:54:57.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:57 smithi059 ceph-mon[21819]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-26T15:54:59.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:54:59 smithi059 ceph-mon[21819]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:54:59.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:54:59 smithi059 ceph-mon[26536]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:54:59.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:54:59 smithi064 ceph-mon[22742]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:55:01.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:01 smithi059 ceph-mon[21819]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:01.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:01.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:01.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:01.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:01.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:01 smithi059 ceph-mon[26536]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:01.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:01.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:01.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:01.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:01.376 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:01 smithi064 ceph-mon[22742]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:01.376 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:01.376 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:01.376 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:01.377 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:03.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:03 smithi059 ceph-mon[21819]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:03.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:03 smithi059 ceph-mon[26536]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:03.387 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:03 smithi064 ceph-mon[22742]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:03.669 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 5 on host 'smithi064' 2024-04-26T15:55:04.266 DEBUG:teuthology.orchestra.run.smithi064:osd.5> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.5.service 2024-04-26T15:55:04.268 INFO:tasks.cephadm:Deploying osd.6 on smithi064 with /dev/vg_nvme/lv_2... 2024-04-26T15:55:04.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T15:55:04.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:04 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:04.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:04 smithi064 ceph-mon[22742]: from='osd.5 [v2:172.21.15.64:6808/1151989321,v1:172.21.15.64:6809/1151989321]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:55:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:04 smithi064 ceph-mon[22742]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:55:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:04 smithi064 ceph-mon[22742]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:04 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:05.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:04 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:05.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:04 smithi059 ceph-mon[21819]: from='osd.5 [v2:172.21.15.64:6808/1151989321,v1:172.21.15.64:6809/1151989321]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:55:05.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:04 smithi059 ceph-mon[21819]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:55:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:04 smithi059 ceph-mon[21819]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:04 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:05.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:04 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:05.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:04 smithi059 ceph-mon[26536]: from='osd.5 [v2:172.21.15.64:6808/1151989321,v1:172.21.15.64:6809/1151989321]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:55:05.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:04 smithi059 ceph-mon[26536]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T15:55:05.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:04 smithi059 ceph-mon[26536]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:05.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:04 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:05.784 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:05 smithi064 ceph-mon[22742]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T15:55:05.784 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:05 smithi064 ceph-mon[22742]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T15:55:05.784 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:05 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:05.784 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:05 smithi064 ceph-mon[22742]: from='osd.5 [v2:172.21.15.64:6808/1151989321,v1:172.21.15.64:6809/1151989321]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:05.784 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:05 smithi064 ceph-mon[22742]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:05.785 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 15:55:05 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[29051]: 2024-04-26T15:55:05.682+0000 7f2c1107f700 -1 osd.5 0 waiting for initial osdmap 2024-04-26T15:55:05.785 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 15:55:05 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[29051]: 2024-04-26T15:55:05.688+0000 7f2c0ca18700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:55:06.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:05 smithi059 ceph-mon[21819]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T15:55:06.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:05 smithi059 ceph-mon[21819]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T15:55:06.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:05 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:06.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:05 smithi059 ceph-mon[21819]: from='osd.5 [v2:172.21.15.64:6808/1151989321,v1:172.21.15.64:6809/1151989321]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:06.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:05 smithi059 ceph-mon[21819]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:06.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:05 smithi059 ceph-mon[26536]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T15:55:06.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:05 smithi059 ceph-mon[26536]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T15:55:06.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:05 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:06.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:05 smithi059 ceph-mon[26536]: from='osd.5 [v2:172.21.15.64:6808/1151989321,v1:172.21.15.64:6809/1151989321]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:06.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:05 smithi059 ceph-mon[26536]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:06.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:06 smithi064 ceph-mon[22742]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:55:06.759 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:06 smithi064 ceph-mon[22742]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T15:55:06.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:06 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:06.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:06 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:06.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:06 smithi064 ceph-mon[22742]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:07.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:06 smithi059 ceph-mon[21819]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:55:07.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:06 smithi059 ceph-mon[21819]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T15:55:07.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:06 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:07.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:06 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:07.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:06 smithi059 ceph-mon[21819]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:07.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:06 smithi059 ceph-mon[26536]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:55:07.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:06 smithi059 ceph-mon[26536]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T15:55:07.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:06 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:07.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:06 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:07.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:06 smithi059 ceph-mon[26536]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-26T15:55:07.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: purged_snaps scrub starts 2024-04-26T15:55:07.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: purged_snaps scrub ok 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: osd.5 [v2:172.21.15.64:6808/1151989321,v1:172.21.15.64:6809/1151989321] boot 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: Detected new or changed devices on smithi064 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: Adjusting osd_memory_target on smithi064 to 8544M 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:07.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:07.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:07 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:08.023 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T15:55:08.023 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T15:55:08.024 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2024-04-26T15:55:08.024 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2024-04-26T15:55:08.024 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012355 s, 849 MB/s 2024-04-26T15:55:08.024 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2024-04-26T15:55:08.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: purged_snaps scrub starts 2024-04-26T15:55:08.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: purged_snaps scrub ok 2024-04-26T15:55:08.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:08.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: osd.5 [v2:172.21.15.64:6808/1151989321,v1:172.21.15.64:6809/1151989321] boot 2024-04-26T15:55:08.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T15:55:08.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:08.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: Detected new or changed devices on smithi064 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: Adjusting osd_memory_target on smithi064 to 8544M 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:08.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: purged_snaps scrub starts 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: purged_snaps scrub ok 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: osd.5 [v2:172.21.15.64:6808/1151989321,v1:172.21.15.64:6809/1151989321] boot 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: Detected new or changed devices on smithi064 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: Adjusting osd_memory_target on smithi064 to 8544M 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:08.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:08.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:08.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:07 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:08.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi064:vg_nvme/lv_2 2024-04-26T15:55:09.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:09 smithi064 ceph-mon[22742]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:09.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:09 smithi064 ceph-mon[22742]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T15:55:09.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:09 smithi064 ceph-mon[22742]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T15:55:09.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:09 smithi059 ceph-mon[21819]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:09.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:09 smithi059 ceph-mon[21819]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T15:55:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:09 smithi059 ceph-mon[21819]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T15:55:09.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:09 smithi059 ceph-mon[26536]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:09.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:09 smithi059 ceph-mon[26536]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T15:55:09.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:09 smithi059 ceph-mon[26536]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T15:55:10.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:10 smithi059 ceph-mon[21819]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T15:55:10.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:10 smithi059 ceph-mon[26536]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T15:55:10.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:10 smithi064 ceph-mon[22742]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T15:55:11.131 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:11 smithi064 ceph-mon[22742]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:11.131 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:11 smithi064 ceph-mon[22742]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T15:55:11.132 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:11 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:55:11.132 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:11 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:55:11.132 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:11 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:11.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:11 smithi059 ceph-mon[21819]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:11.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:11 smithi059 ceph-mon[21819]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T15:55:11.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:55:11.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:55:11.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:11 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:11.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:11 smithi059 ceph-mon[26536]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:11.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:11 smithi059 ceph-mon[26536]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T15:55:11.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:11 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:55:11.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:11 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:55:11.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:11 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:12.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:12 smithi059 ceph-mon[21819]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:55:12.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:12 smithi059 ceph-mon[26536]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:55:12.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:12 smithi064 ceph-mon[22742]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:55:13.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[21819]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:13.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/4166179324' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02410d0f-0d83-403f-ba1c-eb45e7a474be"}]: dispatch 2024-04-26T15:55:13.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02410d0f-0d83-403f-ba1c-eb45e7a474be"}]: dispatch 2024-04-26T15:55:13.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "02410d0f-0d83-403f-ba1c-eb45e7a474be"}]': finished 2024-04-26T15:55:13.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[21819]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T15:55:13.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:13.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[26536]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:13.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/4166179324' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02410d0f-0d83-403f-ba1c-eb45e7a474be"}]: dispatch 2024-04-26T15:55:13.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02410d0f-0d83-403f-ba1c-eb45e7a474be"}]: dispatch 2024-04-26T15:55:13.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "02410d0f-0d83-403f-ba1c-eb45e7a474be"}]': finished 2024-04-26T15:55:13.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[26536]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T15:55:13.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:13 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:13.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:13 smithi064 ceph-mon[22742]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:13.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:13 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/4166179324' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02410d0f-0d83-403f-ba1c-eb45e7a474be"}]: dispatch 2024-04-26T15:55:13.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:13 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02410d0f-0d83-403f-ba1c-eb45e7a474be"}]: dispatch 2024-04-26T15:55:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:13 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "02410d0f-0d83-403f-ba1c-eb45e7a474be"}]': finished 2024-04-26T15:55:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:13 smithi064 ceph-mon[22742]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T15:55:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:13 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:14.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:14 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/2524925333' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:55:14.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:14 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/2524925333' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:55:14.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:14 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/2524925333' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:55:15.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:15 smithi064 ceph-mon[22742]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-04-26T15:55:15.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:15 smithi064 ceph-mon[22742]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T15:55:15.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:15 smithi064 ceph-mon[22742]: Cluster is now healthy 2024-04-26T15:55:15.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:15 smithi059 ceph-mon[26536]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-04-26T15:55:15.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:15 smithi059 ceph-mon[26536]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T15:55:15.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:15 smithi059 ceph-mon[26536]: Cluster is now healthy 2024-04-26T15:55:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:15 smithi059 ceph-mon[21819]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-04-26T15:55:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:15 smithi059 ceph-mon[21819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T15:55:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:15 smithi059 ceph-mon[21819]: Cluster is now healthy 2024-04-26T15:55:17.291 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:17 smithi064 ceph-mon[22742]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-04-26T15:55:17.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:17 smithi059 ceph-mon[26536]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-04-26T15:55:17.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:17 smithi059 ceph-mon[21819]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-04-26T15:55:19.393 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:19 smithi064 ceph-mon[22742]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-26T15:55:19.393 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:19 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T15:55:19.393 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:19 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:19.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:19 smithi059 ceph-mon[21819]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-26T15:55:19.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:19 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T15:55:19.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:19 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:19.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:19 smithi059 ceph-mon[26536]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-26T15:55:19.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:19 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T15:55:19.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:19 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:20.266 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:20 smithi064 ceph-mon[22742]: Deploying daemon osd.6 on smithi064 2024-04-26T15:55:20.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:20 smithi059 ceph-mon[21819]: Deploying daemon osd.6 on smithi064 2024-04-26T15:55:20.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:20 smithi059 ceph-mon[26536]: Deploying daemon osd.6 on smithi064 2024-04-26T15:55:21.421 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:21 smithi064 ceph-mon[22742]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-04-26T15:55:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:21 smithi059 ceph-mon[21819]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-04-26T15:55:21.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:21 smithi059 ceph-mon[26536]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-04-26T15:55:23.403 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:23 smithi064 ceph-mon[22742]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:55:23.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:23 smithi059 ceph-mon[21819]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:55:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:23 smithi059 ceph-mon[26536]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:55:24.919 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:24.920 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:24.920 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:24.920 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:24 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:24.920 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:24 smithi064 ceph-mon[22742]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 46 KiB/s, 0 objects/s recovering 2024-04-26T15:55:25.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:24 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:24 smithi059 ceph-mon[21819]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 46 KiB/s, 0 objects/s recovering 2024-04-26T15:55:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:24 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:24 smithi059 ceph-mon[26536]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 46 KiB/s, 0 objects/s recovering 2024-04-26T15:55:26.353 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 6 on host 'smithi064' 2024-04-26T15:55:27.277 DEBUG:teuthology.orchestra.run.smithi064:osd.6> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.6.service 2024-04-26T15:55:27.279 INFO:tasks.cephadm:Deploying osd.7 on smithi064 with /dev/vg_nvme/lv_1... 2024-04-26T15:55:27.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T15:55:27.503 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:27 smithi064 ceph-mon[22742]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:27.504 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:27 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:27.504 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:27 smithi064 ceph-mon[22742]: from='osd.6 [v2:172.21.15.64:6816/2707336954,v1:172.21.15.64:6817/2707336954]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:55:27.504 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:27 smithi064 ceph-mon[22742]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:55:27.786 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:27 smithi059 ceph-mon[26536]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:27.787 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:27 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:27.787 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:27 smithi059 ceph-mon[26536]: from='osd.6 [v2:172.21.15.64:6816/2707336954,v1:172.21.15.64:6817/2707336954]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:55:27.787 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:27 smithi059 ceph-mon[26536]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:55:27.787 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:27 smithi059 ceph-mon[21819]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:27.787 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:27 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:27.787 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:27 smithi059 ceph-mon[21819]: from='osd.6 [v2:172.21.15.64:6816/2707336954,v1:172.21.15.64:6817/2707336954]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:55:27.787 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:27 smithi059 ceph-mon[21819]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T15:55:28.649 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:28 smithi064 ceph-mon[22742]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T15:55:28.650 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:28 smithi064 ceph-mon[22742]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T15:55:28.650 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:28 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:28.650 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:28 smithi064 ceph-mon[22742]: from='osd.6 [v2:172.21.15.64:6816/2707336954,v1:172.21.15.64:6817/2707336954]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:28.650 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:28 smithi064 ceph-mon[22742]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:28.650 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:28 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:28.650 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 15:55:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[31927]: 2024-04-26T15:55:28.369+0000 7fc596502700 -1 osd.6 0 waiting for initial osdmap 2024-04-26T15:55:28.650 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 15:55:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[31927]: 2024-04-26T15:55:28.376+0000 7fc590e99700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:55:28.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[21819]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T15:55:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[21819]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T15:55:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[21819]: from='osd.6 [v2:172.21.15.64:6816/2707336954,v1:172.21.15.64:6817/2707336954]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[21819]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[26536]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T15:55:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[26536]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T15:55:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[26536]: from='osd.6 [v2:172.21.15.64:6816/2707336954,v1:172.21.15.64:6817/2707336954]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[26536]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:28.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:28 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:29.237 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T15:55:29.238 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T15:55:29.238 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2024-04-26T15:55:29.238 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2024-04-26T15:55:29.238 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0126747 s, 827 MB/s 2024-04-26T15:55:29.238 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2024-04-26T15:55:29.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:29 smithi064 ceph-mon[22742]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:29.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:29 smithi064 ceph-mon[22742]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:55:29.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:29 smithi064 ceph-mon[22742]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T15:55:29.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:29 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:29.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:29 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:29.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:29 smithi059 ceph-mon[21819]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:29.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:29 smithi059 ceph-mon[21819]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:55:29.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:29 smithi059 ceph-mon[21819]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T15:55:29.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:29 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:29.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:29 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:29 smithi059 ceph-mon[26536]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-26T15:55:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:29 smithi059 ceph-mon[26536]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:55:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:29 smithi059 ceph-mon[26536]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T15:55:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:29 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:29 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:29.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi064:vg_nvme/lv_1 2024-04-26T15:55:30.635 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:30 smithi064 ceph-mon[22742]: purged_snaps scrub starts 2024-04-26T15:55:30.635 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:30 smithi064 ceph-mon[22742]: purged_snaps scrub ok 2024-04-26T15:55:30.635 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:30 smithi064 ceph-mon[22742]: osd.6 [v2:172.21.15.64:6816/2707336954,v1:172.21.15.64:6817/2707336954] boot 2024-04-26T15:55:30.635 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:30 smithi064 ceph-mon[22742]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T15:55:30.635 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:30 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:30.635 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:30 smithi064 ceph-mon[22742]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T15:55:30.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[21819]: purged_snaps scrub starts 2024-04-26T15:55:30.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[21819]: purged_snaps scrub ok 2024-04-26T15:55:30.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[21819]: osd.6 [v2:172.21.15.64:6816/2707336954,v1:172.21.15.64:6817/2707336954] boot 2024-04-26T15:55:30.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[21819]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T15:55:30.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[21819]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T15:55:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[26536]: purged_snaps scrub starts 2024-04-26T15:55:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[26536]: purged_snaps scrub ok 2024-04-26T15:55:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[26536]: osd.6 [v2:172.21.15.64:6816/2707336954,v1:172.21.15.64:6817/2707336954] boot 2024-04-26T15:55:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[26536]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T15:55:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:55:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:30 smithi059 ceph-mon[26536]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T15:55:31.563 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:31 smithi064 ceph-mon[22742]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:31.563 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:31 smithi064 ceph-mon[22742]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T15:55:31.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:31 smithi059 ceph-mon[26536]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:31.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:31 smithi059 ceph-mon[26536]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T15:55:31.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:31 smithi059 ceph-mon[21819]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:31.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:31 smithi059 ceph-mon[21819]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T15:55:33.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:33.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T15:55:33.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: Detected new or changed devices on smithi064 2024-04-26T15:55:33.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:33.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:33.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:33.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: Adjusting osd_memory_target on smithi064 to 5696M 2024-04-26T15:55:33.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:33.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:33.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:33.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:33 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: Detected new or changed devices on smithi064 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: Adjusting osd_memory_target on smithi064 to 5696M 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: Detected new or changed devices on smithi064 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: Adjusting osd_memory_target on smithi064 to 5696M 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:33.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:33.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:33 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:34.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:34 smithi064 ceph-mon[22742]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T15:55:34.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:34 smithi064 ceph-mon[22742]: from='client.34289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:55:34.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:34 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:55:34.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:34 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:55:34.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:34 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:34.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:34 smithi059 ceph-mon[21819]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T15:55:34.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:34 smithi059 ceph-mon[21819]: from='client.34289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:55:34.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:34 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:55:34.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:34 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:55:34.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:34 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:34 smithi059 ceph-mon[26536]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T15:55:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:34 smithi059 ceph-mon[26536]: from='client.34289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:55:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:34 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T15:55:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:34 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T15:55:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:34 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:35.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:35 smithi059 ceph-mon[21819]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:35.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:35 smithi059 ceph-mon[26536]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:35 smithi064 ceph-mon[22742]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:36.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:36 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/4068331215' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e21aa790-2584-4a0b-bbfc-340127422fe1"}]: dispatch 2024-04-26T15:55:36.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:36 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e21aa790-2584-4a0b-bbfc-340127422fe1"}]: dispatch 2024-04-26T15:55:36.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:36 smithi059 ceph-mon[21819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e21aa790-2584-4a0b-bbfc-340127422fe1"}]': finished 2024-04-26T15:55:36.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:36 smithi059 ceph-mon[21819]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T15:55:36.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:36 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:36.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:36 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/4068331215' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e21aa790-2584-4a0b-bbfc-340127422fe1"}]: dispatch 2024-04-26T15:55:36.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:36 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e21aa790-2584-4a0b-bbfc-340127422fe1"}]: dispatch 2024-04-26T15:55:36.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:36 smithi059 ceph-mon[26536]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e21aa790-2584-4a0b-bbfc-340127422fe1"}]': finished 2024-04-26T15:55:36.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:36 smithi059 ceph-mon[26536]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T15:55:36.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:36 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:36.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:36 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/4068331215' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e21aa790-2584-4a0b-bbfc-340127422fe1"}]: dispatch 2024-04-26T15:55:36.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:36 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e21aa790-2584-4a0b-bbfc-340127422fe1"}]: dispatch 2024-04-26T15:55:36.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:36 smithi064 ceph-mon[22742]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e21aa790-2584-4a0b-bbfc-340127422fe1"}]': finished 2024-04-26T15:55:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:36 smithi064 ceph-mon[22742]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T15:55:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:36 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:37.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:37 smithi059 ceph-mon[26536]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-26T15:55:37.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:37 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/546874471' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:55:37.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:37 smithi059 ceph-mon[21819]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-26T15:55:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:37 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/546874471' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:55:37.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:37 smithi064 ceph-mon[22742]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-26T15:55:37.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:37 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/546874471' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T15:55:39.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:39 smithi064 ceph-mon[22742]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:55:39.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:39 smithi059 ceph-mon[21819]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:55:39.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:39 smithi059 ceph-mon[26536]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:55:41.304 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:41 smithi064 ceph-mon[22742]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-26T15:55:41.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:41 smithi059 ceph-mon[21819]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-26T15:55:41.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:41 smithi059 ceph-mon[26536]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-26T15:55:42.375 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:42 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T15:55:42.375 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:42 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:42.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:42 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T15:55:42.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:42 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:42.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:42 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T15:55:42.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:42 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:43.273 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:43 smithi064 ceph-mon[22742]: Deploying daemon osd.7 on smithi064 2024-04-26T15:55:43.273 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:43 smithi064 ceph-mon[22742]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-26T15:55:43.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:43 smithi059 ceph-mon[21819]: Deploying daemon osd.7 on smithi064 2024-04-26T15:55:43.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:43 smithi059 ceph-mon[21819]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-26T15:55:43.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:43 smithi059 ceph-mon[26536]: Deploying daemon osd.7 on smithi064 2024-04-26T15:55:43.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:43 smithi059 ceph-mon[26536]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-26T15:55:45.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:45 smithi064 ceph-mon[22742]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:55:45.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:45 smithi059 ceph-mon[21819]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:55:45.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:45 smithi059 ceph-mon[26536]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T15:55:47.283 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:47 smithi064 ceph-mon[22742]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:47.283 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:47.283 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:47.283 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:47.283 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:47 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:47 smithi059 ceph-mon[21819]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:47 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:47.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:47 smithi059 ceph-mon[26536]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:47.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:47.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:47.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:47.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:47 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:49.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:49 smithi064 ceph-mon[22742]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:49.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:49 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:49 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:49.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:49 smithi059 ceph-mon[21819]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:49.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:49 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:49.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:49 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:49.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:49 smithi059 ceph-mon[26536]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:49.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:49 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:49.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:49 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:49.777 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 7 on host 'smithi064' 2024-04-26T15:55:50.391 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:50 smithi064 ceph-mon[22742]: from='osd.7 [v2:172.21.15.64:6824/2061142477,v1:172.21.15.64:6825/2061142477]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:55:50.391 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:50 smithi064 ceph-mon[22742]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:55:50.391 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:50 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:50.391 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:50 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:50.391 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:50 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:50.391 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:50 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:50.417 DEBUG:teuthology.orchestra.run.smithi064:osd.7> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.7.service 2024-04-26T15:55:50.420 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-26T15:55:50.420 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-26T15:55:50.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[21819]: from='osd.7 [v2:172.21.15.64:6824/2061142477,v1:172.21.15.64:6825/2061142477]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:55:50.544 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[21819]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:55:50.545 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:50.545 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:50.545 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:50.545 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:50.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[26536]: from='osd.7 [v2:172.21.15.64:6824/2061142477,v1:172.21.15.64:6825/2061142477]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:55:50.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[26536]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T15:55:50.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:50.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:55:50.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:55:50.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:50 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:55:51.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[26536]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:51.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[26536]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T15:55:51.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[26536]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T15:55:51.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:51.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[26536]: from='osd.7 [v2:172.21.15.64:6824/2061142477,v1:172.21.15.64:6825/2061142477]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:51.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[26536]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:51.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[21819]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:51.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[21819]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T15:55:51.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[21819]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T15:55:51.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:51.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[21819]: from='osd.7 [v2:172.21.15.64:6824/2061142477,v1:172.21.15.64:6825/2061142477]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:51.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:51 smithi059 ceph-mon[21819]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:51.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:51 smithi064 ceph-mon[22742]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:51.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:51 smithi064 ceph-mon[22742]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T15:55:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:51 smithi064 ceph-mon[22742]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T15:55:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:51 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:51 smithi064 ceph-mon[22742]: from='osd.7 [v2:172.21.15.64:6824/2061142477,v1:172.21.15.64:6825/2061142477]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:51 smithi064 ceph-mon[22742]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T15:55:51.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 15:55:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[34915]: 2024-04-26T15:55:51.128+0000 7f7d14262700 -1 osd.7 0 waiting for initial osdmap 2024-04-26T15:55:51.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 15:55:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[34915]: 2024-04-26T15:55:51.137+0000 7f7d0ebf9700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T15:55:52.324 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:55:52.325 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[21819]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:55:52.325 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[21819]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T15:55:52.325 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:52.325 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:52.325 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:55:52.325 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:55:52.325 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[26536]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:55:52.325 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[26536]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T15:55:52.326 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:52.326 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:52.326 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:55:52.326 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:52 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:55:52.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:52 smithi064 ceph-mon[22742]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-04-26T15:55:52.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:52 smithi064 ceph-mon[22742]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T15:55:52.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:52 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:52.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:52 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:52.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:52 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:55:52.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:52 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:55:53.018 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1714146952,"num_in_osds":8,"osd_in_since":1714146935,"num_remapped_pgs":0} 2024-04-26T15:55:53.018 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T15:55:53.296 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:55:53.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[26536]: purged_snaps scrub starts 2024-04-26T15:55:53.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[26536]: purged_snaps scrub ok 2024-04-26T15:55:53.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[26536]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:53.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:53.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[26536]: osd.7 [v2:172.21.15.64:6824/2061142477,v1:172.21.15.64:6825/2061142477] boot 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[26536]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/902664416' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[21819]: purged_snaps scrub starts 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[21819]: purged_snaps scrub ok 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[21819]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[21819]: osd.7 [v2:172.21.15.64:6824/2061142477,v1:172.21.15.64:6825/2061142477] boot 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[21819]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:53 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/902664416' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T15:55:53.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:53 smithi064 ceph-mon[22742]: purged_snaps scrub starts 2024-04-26T15:55:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:53 smithi064 ceph-mon[22742]: purged_snaps scrub ok 2024-04-26T15:55:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:53 smithi064 ceph-mon[22742]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-26T15:55:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:53 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:53 smithi064 ceph-mon[22742]: osd.7 [v2:172.21.15.64:6824/2061142477,v1:172.21.15.64:6825/2061142477] boot 2024-04-26T15:55:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:53 smithi064 ceph-mon[22742]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T15:55:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:53 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:55:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:53 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/902664416' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T15:55:54.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:54 smithi064 ceph-mon[22742]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T15:55:54.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:54 smithi064 ceph-mon[22742]: Detected new or changed devices on smithi064 2024-04-26T15:55:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:54 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:54 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:54 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:54 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:54 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:54 smithi064 ceph-mon[22742]: Adjusting osd_memory_target on smithi064 to 4272M 2024-04-26T15:55:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:54 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:54 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:54.438 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[21819]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[21819]: Detected new or changed devices on smithi064 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[21819]: Adjusting osd_memory_target on smithi064 to 4272M 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[26536]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[26536]: Detected new or changed devices on smithi064 2024-04-26T15:55:54.439 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:54.440 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.440 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.440 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.440 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T15:55:54.440 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[26536]: Adjusting osd_memory_target on smithi064 to 4272M 2024-04-26T15:55:54.440 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:54.440 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:54 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:55:54.915 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:55:54.915 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":57,"fsid":"b6607c98-03e4-11ef-bc93-c7b262605968","created":"2024-04-26T15:51:02.484367+0000","modified":"2024-04-26T15:55:54.135037+0000","last_up_change":"2024-04-26T15:55:52.124714+0000","last_in_change":"2024-04-26T15:55:35.482869+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T15:54:00.288724+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6802","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6803","nonce":1726437871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6804","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6805","nonce":1726437871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6808","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6809","nonce":1726437871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6806","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6807","nonce":1726437871}]},"public_addr":"172.21.15.59:6803/1726437871","cluster_addr":"172.21.15.59:6805/1726437871","heartbeat_back_addr":"172.21.15.59:6809/1726437871","heartbeat_front_addr":"172.21.15.59:6807/1726437871","state":["exists","up"]},{"osd":1,"uuid":"278729e0-965b-44fb-98c5-d5c87036f13d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6810","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6811","nonce":448553573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6812","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6813","nonce":448553573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6816","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6817","nonce":448553573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6814","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6815","nonce":448553573}]},"public_addr":"172.21.15.59:6811/448553573","cluster_addr":"172.21.15.59:6813/448553573","heartbeat_back_addr":"172.21.15.59:6817/448553573","heartbeat_front_addr":"172.21.15.59:6815/448553573","state":["exists","up"]},{"osd":2,"uuid":"bfd237b8-d7ea-4c31-a514-5f499db07cea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6818","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6819","nonce":2282009792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6820","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6821","nonce":2282009792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6824","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6825","nonce":2282009792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6822","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6823","nonce":2282009792}]},"public_addr":"172.21.15.59:6819/2282009792","cluster_addr":"172.21.15.59:6821/2282009792","heartbeat_back_addr":"172.21.15.59:6825/2282009792","heartbeat_front_addr":"172.21.15.59:6823/2282009792","state":["exists","up"]},{"osd":3,"uuid":"58994c22-3506-45b6-a449-81b31271ed94","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6826","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6827","nonce":4167347038}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6828","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6829","nonce":4167347038}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6832","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6833","nonce":4167347038}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6830","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6831","nonce":4167347038}]},"public_addr":"172.21.15.59:6827/4167347038","cluster_addr":"172.21.15.59:6829/4167347038","heartbeat_back_addr":"172.21.15.59:6833/4167347038","heartbeat_front_addr":"172.21.15.59:6831/4167347038","state":["exists","up"]},{"osd":4,"uuid":"193191de-111e-4026-a251-a4a1f9a7a8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6801","nonce":304831184}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6803","nonce":304831184}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6807","nonce":304831184}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6805","nonce":304831184}]},"public_addr":"172.21.15.64:6801/304831184","cluster_addr":"172.21.15.64:6803/304831184","heartbeat_back_addr":"172.21.15.64:6807/304831184","heartbeat_front_addr":"172.21.15.64:6805/304831184","state":["exists","up"]},{"osd":5,"uuid":"601697da-23df-4269-a707-eb9435930d2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6809","nonce":1151989321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6811","nonce":1151989321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6815","nonce":1151989321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6813","nonce":1151989321}]},"public_addr":"172.21.15.64:6809/1151989321","cluster_addr":"172.21.15.64:6811/1151989321","heartbeat_back_addr":"172.21.15.64:6815/1151989321","heartbeat_front_addr":"172.21.15.64:6813/1151989321","state":["exists","up"]},{"osd":6,"uuid":"02410d0f-0d83-403f-ba1c-eb45e7a474be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6817","nonce":2707336954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6819","nonce":2707336954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6823","nonce":2707336954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6821","nonce":2707336954}]},"public_addr":"172.21.15.64:6817/2707336954","cluster_addr":"172.21.15.64:6819/2707336954","heartbeat_back_addr":"172.21.15.64:6823/2707336954","heartbeat_front_addr":"172.21.15.64:6821/2707336954","state":["exists","up"]},{"osd":7,"uuid":"e21aa790-2584-4a0b-bbfc-340127422fe1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6825","nonce":2061142477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6827","nonce":2061142477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6831","nonce":2061142477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6829","nonce":2061142477}]},"public_addr":"172.21.15.64:6825/2061142477","cluster_addr":"172.21.15.64:6827/2061142477","heartbeat_back_addr":"172.21.15.64:6831/2061142477","heartbeat_front_addr":"172.21.15.64:6829/2061142477","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:53:08.329520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:53:31.622804+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:53:56.343217+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:54:20.342834+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:54:41.812004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:55:04.787063+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:55:27.741181+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:55:50.663035+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.59:0/635818739":"2024-04-27T15:51:51.729465+0000","172.21.15.59:6801/1212694920":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/1573688154":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/1094121986":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/1707008100":"2024-04-27T15:51:29.308166+0000","172.21.15.59:6800/890201972":"2024-04-27T15:51:29.308166+0000","172.21.15.59:6800/1212694920":"2024-04-27T15:51:51.729465+0000","172.21.15.59:6801/890201972":"2024-04-27T15:51:29.308166+0000","172.21.15.59:0/3060676991":"2024-04-27T15:51:29.308166+0000","172.21.15.59:0/2924841679":"2024-04-27T15:51:29.308166+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T15:55:55.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:55 smithi064 ceph-mon[22742]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:55:55.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:55 smithi064 ceph-mon[22742]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T15:55:55.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:55 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/164535639' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:55:55.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:55 smithi059 ceph-mon[26536]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:55:55.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:55 smithi059 ceph-mon[26536]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T15:55:55.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:55 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/164535639' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:55:55.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:55 smithi059 ceph-mon[21819]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:55:55.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:55 smithi059 ceph-mon[21819]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T15:55:55.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:55 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/164535639' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:55:55.645 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-26T15:54:00.288724+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-26T15:55:55.646 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-26T15:55:55.873 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:55:56.303 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:56 smithi059 ceph-mon[26536]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T15:55:56.303 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:56 smithi059 ceph-mon[21819]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T15:55:56.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:56 smithi064 ceph-mon[22742]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T15:55:57.243 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:57 smithi059 ceph-mon[21819]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:55:57.243 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:57 smithi059 ceph-mon[21819]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T15:55:57.244 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:57 smithi059 ceph-mon[26536]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:55:57.244 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:57 smithi059 ceph-mon[26536]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T15:55:57.636 INFO:teuthology.orchestra.run.smithi059.stdout:pg_num: 1 2024-04-26T15:55:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:57 smithi064 ceph-mon[22742]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:55:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:57 smithi064 ceph-mon[22742]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T15:55:58.233 INFO:tasks.cephadm:Adding prometheus.a on smithi064 2024-04-26T15:55:58.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi064=a' 2024-04-26T15:55:58.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:58 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3553846770' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T15:55:58.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:58 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3553846770' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T15:55:58.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:58 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3553846770' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T15:55:59.549 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:55:59 smithi064 ceph-mon[22742]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:55:59.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:55:59 smithi059 ceph-mon[21819]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:55:59.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:55:59 smithi059 ceph-mon[26536]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:00.144 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled prometheus update... 2024-04-26T15:56:00.699 DEBUG:teuthology.orchestra.run.smithi064:prometheus.a> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-26T15:56:00.705 INFO:tasks.cephadm:Adding node-exporter.a on smithi059 2024-04-26T15:56:00.705 INFO:tasks.cephadm:Adding node-exporter.b on smithi064 2024-04-26T15:56:00.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi059=a;smithi064=b' 2024-04-26T15:56:01.362 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ignoring --setuser ceph since I am not root 2024-04-26T15:56:01.363 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ignoring --setgroup ceph since I am not root 2024-04-26T15:56:01.363 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-mgr[23926]: -- 172.21.15.64:0/2440957467 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561b8ebe4340 con 0x561b8ebbcc00 2024-04-26T15:56:01.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-mon[22742]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:56:01.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-mon[22742]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:01.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-mon[22742]: Saving service prometheus spec with placement smithi064=a;count:1 2024-04-26T15:56:01.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:56:01.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:01.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:56:01.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:56:01.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:56:01.364 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T15:56:01.510 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[21819]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[21819]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[21819]: Saving service prometheus spec with placement smithi064=a;count:1 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ignoring --setuser ceph since I am not root 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ignoring --setgroup ceph since I am not root 2024-04-26T15:56:01.511 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mgr[22056]: -- 172.21.15.59:0/2410318567 <== mon.1 v2:172.21.15.59:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5646aba02340 con 0x5646ab9da800 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:01.377+0000 7f2e1ce08000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:01.510+0000 7f2e1ce08000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[26536]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[26536]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[26536]: Saving service prometheus spec with placement smithi064=a;count:1 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' 2024-04-26T15:56:01.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T15:56:01.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:01.371+0000 7f84705df000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T15:56:01.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:01.506+0000 7f84705df000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:56:01.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:01.607+0000 7f84705df000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T15:56:01.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:01.612+0000 7f2e1ce08000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T15:56:02.021 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:01.932+0000 7f84705df000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T15:56:02.204 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:01.930+0000 7f2e1ce08000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T15:56:02.204 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:02.015+0000 7f2e1ce08000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T15:56:02.327 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:02 smithi064 ceph-mon[22742]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T15:56:02.327 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:02 smithi064 ceph-mon[22742]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T15:56:02.327 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:02.021+0000 7f84705df000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T15:56:02.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:02 smithi059 ceph-mon[21819]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T15:56:02.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:02 smithi059 ceph-mon[21819]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T15:56:02.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:02 smithi059 ceph-mon[26536]: from='mgr.14150 172.21.15.59:0/3286137045' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T15:56:02.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:02 smithi059 ceph-mon[26536]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T15:56:03.454 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:03.196+0000 7f2e1ce08000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T15:56:03.454 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:03.366+0000 7f2e1ce08000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T15:56:03.455 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:03.451+0000 7f2e1ce08000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T15:56:03.458 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:03.204+0000 7f84705df000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T15:56:03.458 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:03.374+0000 7f84705df000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T15:56:03.458 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:03.459+0000 7f84705df000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T15:56:04.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:03.775+0000 7f2e1ce08000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T15:56:04.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:03.870+0000 7f2e1ce08000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T15:56:04.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:03.967+0000 7f2e1ce08000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T15:56:04.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:03.781+0000 7f84705df000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T15:56:04.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:03.874+0000 7f84705df000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T15:56:04.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:03.970+0000 7f84705df000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T15:56:04.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:04.217+0000 7f2e1ce08000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:56:04.676 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:04.218+0000 7f84705df000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T15:56:04.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:04.676+0000 7f84705df000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T15:56:05.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:04.677+0000 7f2e1ce08000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T15:56:05.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:05 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:05.176+0000 7f84705df000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T15:56:05.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:05 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:05.272+0000 7f84705df000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T15:56:05.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:05.183+0000 7f2e1ce08000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T15:56:05.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:05.280+0000 7f2e1ce08000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T15:56:05.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:05 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:05.576+0000 7f84705df000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T15:56:05.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:05 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:05.668+0000 7f84705df000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T15:56:06.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:05.587+0000 7f2e1ce08000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T15:56:06.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:05.680+0000 7f2e1ce08000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T15:56:06.431 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:06.083+0000 7f84705df000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T15:56:06.431 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:06.219+0000 7f84705df000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T15:56:06.431 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:06.314+0000 7f84705df000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T15:56:06.431 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:06.430+0000 7f84705df000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T15:56:06.454 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:06.101+0000 7f2e1ce08000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T15:56:06.454 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:06.238+0000 7f2e1ce08000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T15:56:06.454 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:06.335+0000 7f2e1ce08000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T15:56:06.454 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:06.454+0000 7f2e1ce08000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T15:56:06.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:06.522+0000 7f84705df000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T15:56:06.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:06.547+0000 7f2e1ce08000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T15:56:07.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:07.089+0000 7f84705df000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T15:56:07.493 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:07.121+0000 7f2e1ce08000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T15:56:07.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:07.493+0000 7f2e1ce08000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T15:56:07.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:07.458+0000 7f84705df000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T15:56:07.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: [26/Apr/2024:15:56:07] ENGINE Bus STARTING 2024-04-26T15:56:07.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: CherryPy Checker: 2024-04-26T15:56:07.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: The Application mounted at '' has an empty config. 2024-04-26T15:56:07.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: 2024-04-26T15:56:07.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: [26/Apr/2024:15:56:07] ENGINE Serving on http://:::9283 2024-04-26T15:56:07.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: [26/Apr/2024:15:56:07] ENGINE Bus STARTED 2024-04-26T15:56:08.312 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:56:08] ENGINE Bus STARTING 2024-04-26T15:56:08.312 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: CherryPy Checker: 2024-04-26T15:56:08.312 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: The Application mounted at '' has an empty config. 2024-04-26T15:56:08.312 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:56:08] ENGINE Serving on http://:::9283 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:56:08] ENGINE Bus STARTED 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[26536]: Standby manager daemon x restarted 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[26536]: Standby manager daemon x started 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[26536]: Active manager daemon y restarted 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[26536]: Activating manager daemon y 2024-04-26T15:56:08.313 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[26536]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T15:56:08.314 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[21819]: Standby manager daemon x restarted 2024-04-26T15:56:08.314 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[21819]: Standby manager daemon x started 2024-04-26T15:56:08.314 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T15:56:08.314 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:56:08.314 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T15:56:08.314 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:56:08.314 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[21819]: Active manager daemon y restarted 2024-04-26T15:56:08.314 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[21819]: Activating manager daemon y 2024-04-26T15:56:08.314 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:08 smithi059 ceph-mon[21819]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T15:56:08.419 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:08 smithi064 ceph-mon[22742]: Standby manager daemon x restarted 2024-04-26T15:56:08.419 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:08 smithi064 ceph-mon[22742]: Standby manager daemon x started 2024-04-26T15:56:08.419 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:08 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T15:56:08.419 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:08 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T15:56:08.419 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:08 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T15:56:08.419 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:08 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/2340253236' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T15:56:08.419 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:08 smithi064 ceph-mon[22742]: Active manager daemon y restarted 2024-04-26T15:56:08.419 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:08 smithi064 ceph-mon[22742]: Activating manager daemon y 2024-04-26T15:56:08.419 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:08 smithi064 ceph-mon[22742]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T15:56:09.088 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled node-exporter update... 2024-04-26T15:56:09.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: mgrmap e17: y(active, starting, since 0.573736s), standbys: x 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:56:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: Manager daemon y is now available 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:56:09.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:56:09.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:56:09.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:56:09.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:56:09] ENGINE Bus STARTING 2024-04-26T15:56:09.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:56:09] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T15:56:09.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:15:56:09] ENGINE Bus STARTED 2024-04-26T15:56:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: mgrmap e17: y(active, starting, since 0.573736s), standbys: x 2024-04-26T15:56:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:56:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:56:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:56:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T15:56:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: Manager daemon y is now available 2024-04-26T15:56:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: mgrmap e17: y(active, starting, since 0.573736s), standbys: x 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T15:56:09.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T15:56:09.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: Manager daemon y is now available 2024-04-26T15:56:09.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:09.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:09.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:56:09.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:56:09.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:09.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:56:09.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:56:09.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:56:09.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:56:10.093 DEBUG:teuthology.orchestra.run.smithi059:node-exporter.a> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-26T15:56:10.096 DEBUG:teuthology.orchestra.run.smithi064:node-exporter.b> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-26T15:56:10.098 INFO:tasks.cephadm:Adding alertmanager.a on smithi059 2024-04-26T15:56:10.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi059=a' 2024-04-26T15:56:10.255 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[26536]: [26/Apr/2024:15:56:09] ENGINE Bus STARTING 2024-04-26T15:56:10.255 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[26536]: mgrmap e18: y(active, since 1.57773s), standbys: x 2024-04-26T15:56:10.255 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[26536]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi059=a;smithi064=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:10.255 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[26536]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:10.255 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[26536]: Saving service node-exporter spec with placement smithi059=a;smithi064=b;count:2 2024-04-26T15:56:10.255 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:10.255 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[26536]: [26/Apr/2024:15:56:09] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T15:56:10.255 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[26536]: [26/Apr/2024:15:56:09] ENGINE Bus STARTED 2024-04-26T15:56:10.255 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:10.255 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:10.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[21819]: [26/Apr/2024:15:56:09] ENGINE Bus STARTING 2024-04-26T15:56:10.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[21819]: mgrmap e18: y(active, since 1.57773s), standbys: x 2024-04-26T15:56:10.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[21819]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi059=a;smithi064=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:10.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[21819]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:10.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[21819]: Saving service node-exporter spec with placement smithi059=a;smithi064=b;count:2 2024-04-26T15:56:10.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:10.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[21819]: [26/Apr/2024:15:56:09] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T15:56:10.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[21819]: [26/Apr/2024:15:56:09] ENGINE Bus STARTED 2024-04-26T15:56:10.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:10.256 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:10 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:10.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:10 smithi064 ceph-mon[22742]: [26/Apr/2024:15:56:09] ENGINE Bus STARTING 2024-04-26T15:56:10.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:10 smithi064 ceph-mon[22742]: mgrmap e18: y(active, since 1.57773s), standbys: x 2024-04-26T15:56:10.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:10 smithi064 ceph-mon[22742]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi059=a;smithi064=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:10.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:10 smithi064 ceph-mon[22742]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:10.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:10 smithi064 ceph-mon[22742]: Saving service node-exporter spec with placement smithi059=a;smithi064=b;count:2 2024-04-26T15:56:10.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:10 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:10.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:10 smithi064 ceph-mon[22742]: [26/Apr/2024:15:56:09] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T15:56:10.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:10 smithi064 ceph-mon[22742]: [26/Apr/2024:15:56:09] ENGINE Bus STARTED 2024-04-26T15:56:10.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:10 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:10.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:10 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.289 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:11 smithi064 ceph-mon[22742]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:11.290 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:11 smithi064 ceph-mon[22742]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T15:56:11.290 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:11 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.290 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:11 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T15:56:11.290 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:11 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.290 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:11 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.290 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:11 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[21819]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:11.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[21819]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[26536]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[26536]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:11 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:12.144 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled alertmanager update... 2024-04-26T15:56:12.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:12 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:56:12.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:12 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T15:56:12.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:12 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:56:12.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:12 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:12.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:12 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:12.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:12 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:12.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:56:12.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T15:56:12.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:56:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:56:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T15:56:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T15:56:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:12 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:12.944 DEBUG:teuthology.orchestra.run.smithi059:alertmanager.a> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-26T15:56:12.948 INFO:tasks.cephadm:Adding grafana.a on smithi064 2024-04-26T15:56:12.948 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi064=a' 2024-04-26T15:56:13.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:13 smithi059 ceph-mon[26536]: Deploying daemon prometheus.a on smithi064 2024-04-26T15:56:13.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:13 smithi059 ceph-mon[26536]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:13.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:13 smithi059 ceph-mon[26536]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi059=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:13.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:13 smithi059 ceph-mon[26536]: Saving service alertmanager spec with placement smithi059=a;count:1 2024-04-26T15:56:13.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:13 smithi059 ceph-mon[21819]: Deploying daemon prometheus.a on smithi064 2024-04-26T15:56:13.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:13 smithi059 ceph-mon[21819]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:13.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:13 smithi059 ceph-mon[21819]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi059=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:13.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:13 smithi059 ceph-mon[21819]: Saving service alertmanager spec with placement smithi059=a;count:1 2024-04-26T15:56:13.595 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:13 smithi064 ceph-mon[22742]: Deploying daemon prometheus.a on smithi064 2024-04-26T15:56:13.595 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:13 smithi064 ceph-mon[22742]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:13.595 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:13 smithi064 ceph-mon[22742]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi059=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:13.595 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:13 smithi064 ceph-mon[22742]: Saving service alertmanager spec with placement smithi059=a;count:1 2024-04-26T15:56:15.405 INFO:teuthology.orchestra.run.smithi064.stdout:Scheduled grafana update... 2024-04-26T15:56:15.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:15 smithi059 ceph-mon[21819]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:15 smithi059 ceph-mon[26536]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:15 smithi064 ceph-mon[22742]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:16.345 DEBUG:teuthology.orchestra.run.smithi064:grafana.a> sudo journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@grafana.a.service 2024-04-26T15:56:16.348 INFO:tasks.cephadm:Setting up client nodes... 2024-04-26T15:56:16.348 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T15:56:16.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:16 smithi059 ceph-mon[26536]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:16.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:16 smithi059 ceph-mon[26536]: Saving service grafana spec with placement smithi064=a;count:1 2024-04-26T15:56:16.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:16 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:16.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:16 smithi059 ceph-mon[26536]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:16.672 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:16 smithi059 ceph-mon[21819]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:16.672 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:16 smithi059 ceph-mon[21819]: Saving service grafana spec with placement smithi064=a;count:1 2024-04-26T15:56:16.672 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:16 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:16.672 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:16 smithi059 ceph-mon[21819]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:16 smithi064 ceph-mon[22742]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:56:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:16 smithi064 ceph-mon[22742]: Saving service grafana spec with placement smithi064=a;count:1 2024-04-26T15:56:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:16 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:16 smithi064 ceph-mon[22742]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:18.640 INFO:teuthology.orchestra.run.smithi059.stdout:[client.0] 2024-04-26T15:56:18.640 INFO:teuthology.orchestra.run.smithi059.stdout: key = AQCizitmPYPnJRAAWlJFf/KaN5IvJM0BTgf40A== 2024-04-26T15:56:19.174 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-26T15:56:19.174 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-26T15:56:19.174 DEBUG:teuthology.orchestra.run.smithi059:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-26T15:56:19.197 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:19 smithi059 ceph-mon[21819]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:19.197 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:19 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1101532357' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:19.197 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:19 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:19.197 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:19 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T15:56:19.198 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:19 smithi059 ceph-mon[26536]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:19.198 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:19 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1101532357' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:19.198 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:19 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:19.198 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:19 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T15:56:19.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T15:56:19.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:19 smithi064 ceph-mon[22742]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:19.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:19 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1101532357' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:19 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:19 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T15:56:21.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:21 smithi064 ceph-mon[22742]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:21.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:21 smithi059 ceph-mon[26536]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:21 smithi059 ceph-mon[21819]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:22.384 INFO:teuthology.orchestra.run.smithi064.stdout:[client.1] 2024-04-26T15:56:22.385 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQCmzitm/h2SFhAAtdc4JKUyV+zvPQ0yCwZRNg== 2024-04-26T15:56:23.037 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-04-26T15:56:23.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-26T15:56:23.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-26T15:56:23.078 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T15:56:23.078 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T15:56:23.078 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T15:56:23.274 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:23 smithi064 ceph-mon[22742]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:23.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:23 smithi064 ceph-mon[22742]: from='client.? 172.21.15.64:0/3333598430' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:23.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:23 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:23.275 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:23 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T15:56:23.311 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:23.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:23 smithi059 ceph-mon[26536]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:23.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:23 smithi059 ceph-mon[26536]: from='client.? 172.21.15.64:0/3333598430' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:23.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:23 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:23.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:23 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T15:56:23.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:23 smithi059 ceph-mon[21819]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:23.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:23 smithi059 ceph-mon[21819]: from='client.? 172.21.15.64:0/3333598430' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:23.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:23 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T15:56:23.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:23 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T15:56:24.473 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:24 smithi064 systemd[1]: Starting Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:56:24.936 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:24 smithi064 podman[37828]: 2024-04-26 15:56:24.600077262 +0000 UTC m=+0.019440158 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-26T15:56:25.185 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:56:25.339 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:25 smithi059 ceph-mon[26536]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:25.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:25 smithi059 ceph-mon[21819]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:25.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-mon[22742]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:25.755 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 podman[37828]: 2024-04-26 15:56:25.500359974 +0000 UTC m=+0.919722854 container create 91f5db4c48b21e6f0d15f2237893369dd8398bb6387ee28571db99b5e39d8629 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T15:56:25.755 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 podman[37828]: 2024-04-26 15:56:25.72478562 +0000 UTC m=+1.144148501 container init 91f5db4c48b21e6f0d15f2237893369dd8398bb6387ee28571db99b5e39d8629 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T15:56:25.756 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 podman[37828]: 2024-04-26 15:56:25.728181461 +0000 UTC m=+1.147544344 container start 91f5db4c48b21e6f0d15f2237893369dd8398bb6387ee28571db99b5e39d8629 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T15:56:25.778 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":19,"active_gid":24352,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6800","nonce":3669944706},{"type":"v1","addr":"172.21.15.59:6801","nonce":3669944706}]},"active_addr":"172.21.15.59:6801/3669944706","active_change":"2024-04-26T15:56:07.497019+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":34340,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.59:8443/","prometheus":"http://172.21.15.59:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.59:0","nonce":4008180956}]},{"addrvec":[{"type":"v2","addr":"172.21.15.59:0","nonce":144595486}]},{"addrvec":[{"type":"v2","addr":"172.21.15.59:0","nonce":1390537899}]},{"addrvec":[{"type":"v2","addr":"172.21.15.59:0","nonce":3519778281}]}]}} 2024-04-26T15:56:25.784 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T15:56:25.784 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T15:56:25.784 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T15:56:26.017 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:26.135 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.756Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-26T15:56:26.135 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.756Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-26T15:56:26.135 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.756Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.756Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi064 (none))" 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.756Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.756Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.758Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.759Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.759Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.762Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.762Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.937µs 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.762Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.763Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.763Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=26.795µs wal_replay_duration=446.162µs total_replay_duration=488.521µs 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.763Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T15:56:26.136 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.763Z caller=main.go:947 level=info msg="TSDB started" 2024-04-26T15:56:26.137 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.763Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T15:56:26.137 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.780Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.558312ms db_storage=741ns remote_storage=1.782µs web_handler=311ns query_engine=732ns scrape=410.337µs scrape_sd=18.684µs notify=1.07µs notify_sd=1.445µs rules=15.947636ms 2024-04-26T15:56:26.137 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:56:25.780Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-26T15:56:26.137 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 bash[37828]: 91f5db4c48b21e6f0d15f2237893369dd8398bb6387ee28571db99b5e39d8629 2024-04-26T15:56:26.137 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:56:25 smithi064 systemd[1]: Started Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:56:26.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:26 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2115838853' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T15:56:26.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:26 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:26.458 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:26 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2115838853' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T15:56:26.459 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:26 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:26.459 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:26 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2115838853' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T15:56:26.459 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:26 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:27.134 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:27 smithi059 systemd[1]: Starting Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:56:27.411 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:27 smithi059 ceph-mon[21819]: Deploying daemon node-exporter.a on smithi059 2024-04-26T15:56:27.412 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:27 smithi059 ceph-mon[21819]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:27.412 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:27 smithi059 ceph-mon[26536]: Deploying daemon node-exporter.a on smithi059 2024-04-26T15:56:27.412 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:27 smithi059 ceph-mon[26536]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:27.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:27 smithi064 ceph-mon[22742]: Deploying daemon node-exporter.a on smithi059 2024-04-26T15:56:27.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:27 smithi064 ceph-mon[22742]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:27.961 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:56:27.961 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":60,"fsid":"b6607c98-03e4-11ef-bc93-c7b262605968","created":"2024-04-26T15:51:02.484367+0000","modified":"2024-04-26T15:56:07.496640+0000","last_up_change":"2024-04-26T15:55:52.124714+0000","last_in_change":"2024-04-26T15:55:35.482869+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T15:54:00.288724+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6802","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6803","nonce":1726437871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6804","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6805","nonce":1726437871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6808","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6809","nonce":1726437871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6806","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6807","nonce":1726437871}]},"public_addr":"172.21.15.59:6803/1726437871","cluster_addr":"172.21.15.59:6805/1726437871","heartbeat_back_addr":"172.21.15.59:6809/1726437871","heartbeat_front_addr":"172.21.15.59:6807/1726437871","state":["exists","up"]},{"osd":1,"uuid":"278729e0-965b-44fb-98c5-d5c87036f13d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6810","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6811","nonce":448553573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6812","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6813","nonce":448553573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6816","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6817","nonce":448553573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6814","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6815","nonce":448553573}]},"public_addr":"172.21.15.59:6811/448553573","cluster_addr":"172.21.15.59:6813/448553573","heartbeat_back_addr":"172.21.15.59:6817/448553573","heartbeat_front_addr":"172.21.15.59:6815/448553573","state":["exists","up"]},{"osd":2,"uuid":"bfd237b8-d7ea-4c31-a514-5f499db07cea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6818","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6819","nonce":2282009792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6820","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6821","nonce":2282009792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6824","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6825","nonce":2282009792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6822","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6823","nonce":2282009792}]},"public_addr":"172.21.15.59:6819/2282009792","cluster_addr":"172.21.15.59:6821/2282009792","heartbeat_back_addr":"172.21.15.59:6825/2282009792","heartbeat_front_addr":"172.21.15.59:6823/2282009792","state":["exists","up"]},{"osd":3,"uuid":"58994c22-3506-45b6-a449-81b31271ed94","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6826","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6827","nonce":4167347038}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6828","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6829","nonce":4167347038}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6832","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6833","nonce":4167347038}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6830","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6831","nonce":4167347038}]},"public_addr":"172.21.15.59:6827/4167347038","cluster_addr":"172.21.15.59:6829/4167347038","heartbeat_back_addr":"172.21.15.59:6833/4167347038","heartbeat_front_addr":"172.21.15.59:6831/4167347038","state":["exists","up"]},{"osd":4,"uuid":"193191de-111e-4026-a251-a4a1f9a7a8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6801","nonce":304831184}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6803","nonce":304831184}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6807","nonce":304831184}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6805","nonce":304831184}]},"public_addr":"172.21.15.64:6801/304831184","cluster_addr":"172.21.15.64:6803/304831184","heartbeat_back_addr":"172.21.15.64:6807/304831184","heartbeat_front_addr":"172.21.15.64:6805/304831184","state":["exists","up"]},{"osd":5,"uuid":"601697da-23df-4269-a707-eb9435930d2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6809","nonce":1151989321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6811","nonce":1151989321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6815","nonce":1151989321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6813","nonce":1151989321}]},"public_addr":"172.21.15.64:6809/1151989321","cluster_addr":"172.21.15.64:6811/1151989321","heartbeat_back_addr":"172.21.15.64:6815/1151989321","heartbeat_front_addr":"172.21.15.64:6813/1151989321","state":["exists","up"]},{"osd":6,"uuid":"02410d0f-0d83-403f-ba1c-eb45e7a474be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6817","nonce":2707336954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6819","nonce":2707336954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6823","nonce":2707336954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6821","nonce":2707336954}]},"public_addr":"172.21.15.64:6817/2707336954","cluster_addr":"172.21.15.64:6819/2707336954","heartbeat_back_addr":"172.21.15.64:6823/2707336954","heartbeat_front_addr":"172.21.15.64:6821/2707336954","state":["exists","up"]},{"osd":7,"uuid":"e21aa790-2584-4a0b-bbfc-340127422fe1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6825","nonce":2061142477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6827","nonce":2061142477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6831","nonce":2061142477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6829","nonce":2061142477}]},"public_addr":"172.21.15.64:6825/2061142477","cluster_addr":"172.21.15.64:6827/2061142477","heartbeat_back_addr":"172.21.15.64:6831/2061142477","heartbeat_front_addr":"172.21.15.64:6829/2061142477","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:53:08.329520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:53:31.622804+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:53:56.343217+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:54:20.342834+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:54:41.812004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:55:04.787063+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:55:27.741181+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:55:50.663035+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.59:6800/434932617":"2024-04-27T15:56:07.496595+0000","172.21.15.59:0/2069431914":"2024-04-27T15:56:07.496595+0000","172.21.15.59:0/3047449642":"2024-04-27T15:56:07.496595+0000","172.21.15.59:0/635818739":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/3932195089":"2024-04-27T15:56:07.496595+0000","172.21.15.59:6801/1212694920":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/1573688154":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/2655313575":"2024-04-27T15:56:07.496595+0000","172.21.15.59:0/1094121986":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/1707008100":"2024-04-27T15:51:29.308166+0000","172.21.15.59:6800/890201972":"2024-04-27T15:51:29.308166+0000","172.21.15.59:6800/1212694920":"2024-04-27T15:51:51.729465+0000","172.21.15.59:6801/890201972":"2024-04-27T15:51:29.308166+0000","172.21.15.59:6801/434932617":"2024-04-27T15:56:07.496595+0000","172.21.15.59:0/3060676991":"2024-04-27T15:51:29.308166+0000","172.21.15.59:0/2924841679":"2024-04-27T15:51:29.308166+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T15:56:28.083 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:27 smithi059 bash[41472]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-26T15:56:28.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:28 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2259698583' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:56:28.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:28 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:28.571 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:28 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2259698583' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:56:28.571 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:28 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:28.572 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:28 smithi059 bash[41472]: Getting image source signatures 2024-04-26T15:56:28.572 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:28 smithi059 bash[41472]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-26T15:56:28.572 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:28 smithi059 bash[41472]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-26T15:56:28.572 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:28 smithi059 bash[41472]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-26T15:56:28.572 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:28 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2259698583' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:56:28.572 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:28 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:28.596 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T15:56:28.596 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T15:56:28.830 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:29.214 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:29 smithi059 ceph-mon[21819]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:29.214 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:29 smithi059 ceph-mon[26536]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:29.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:29 smithi064 ceph-mon[22742]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:30.582 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:30 smithi059 bash[41472]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-26T15:56:30.582 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:30 smithi059 bash[41472]: Writing manifest to image destination 2024-04-26T15:56:31.051 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:30 smithi059 podman[41472]: 2024-04-26 15:56:30.73104732 +0000 UTC m=+2.922247499 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-26T15:56:31.051 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:30 smithi059 podman[41472]: 2024-04-26 15:56:30.946355747 +0000 UTC m=+3.137555916 container create ae5b4f0c81d44fb2c410691503121ccfbc862eb81b70eb86313bc260c61f1be1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T15:56:31.423 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-mon[26536]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:31.424 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-mon[21819]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:31.424 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 podman[41472]: 2024-04-26 15:56:31.148715945 +0000 UTC m=+3.339916112 container init ae5b4f0c81d44fb2c410691503121ccfbc862eb81b70eb86313bc260c61f1be1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T15:56:31.424 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 podman[41472]: 2024-04-26 15:56:31.152339316 +0000 UTC m=+3.343539490 container start ae5b4f0c81d44fb2c410691503121ccfbc862eb81b70eb86313bc260c61f1be1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T15:56:31.424 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.157Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-26T15:56:31.424 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.157Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-26T15:56:31.424 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-26T15:56:31.424 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T15:56:31.424 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-26T15:56:31.425 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=arp 2024-04-26T15:56:31.425 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-26T15:56:31.425 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-26T15:56:31.425 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-26T15:56:31.425 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-26T15:56:31.425 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-26T15:56:31.425 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-26T15:56:31.425 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-26T15:56:31.425 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-26T15:56:31.425 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=edac 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-26T15:56:31.426 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=os 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=stat 2024-04-26T15:56:31.427 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=time 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=uname 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.158Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[41635]: ts=2024-04-26T15:56:31.159Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 bash[41472]: ae5b4f0c81d44fb2c410691503121ccfbc862eb81b70eb86313bc260c61f1be1 2024-04-26T15:56:31.428 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 15:56:31 smithi059 systemd[1]: Started Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:56:31.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:31 smithi064 ceph-mon[22742]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:31.945 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:56:31.945 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":60,"fsid":"b6607c98-03e4-11ef-bc93-c7b262605968","created":"2024-04-26T15:51:02.484367+0000","modified":"2024-04-26T15:56:07.496640+0000","last_up_change":"2024-04-26T15:55:52.124714+0000","last_in_change":"2024-04-26T15:55:35.482869+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T15:54:00.288724+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"aaf91f18-9f8a-46e2-8fdf-22e4cf4f1dda","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6802","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6803","nonce":1726437871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6804","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6805","nonce":1726437871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6808","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6809","nonce":1726437871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6806","nonce":1726437871},{"type":"v1","addr":"172.21.15.59:6807","nonce":1726437871}]},"public_addr":"172.21.15.59:6803/1726437871","cluster_addr":"172.21.15.59:6805/1726437871","heartbeat_back_addr":"172.21.15.59:6809/1726437871","heartbeat_front_addr":"172.21.15.59:6807/1726437871","state":["exists","up"]},{"osd":1,"uuid":"278729e0-965b-44fb-98c5-d5c87036f13d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6810","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6811","nonce":448553573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6812","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6813","nonce":448553573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6816","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6817","nonce":448553573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6814","nonce":448553573},{"type":"v1","addr":"172.21.15.59:6815","nonce":448553573}]},"public_addr":"172.21.15.59:6811/448553573","cluster_addr":"172.21.15.59:6813/448553573","heartbeat_back_addr":"172.21.15.59:6817/448553573","heartbeat_front_addr":"172.21.15.59:6815/448553573","state":["exists","up"]},{"osd":2,"uuid":"bfd237b8-d7ea-4c31-a514-5f499db07cea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6818","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6819","nonce":2282009792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6820","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6821","nonce":2282009792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6824","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6825","nonce":2282009792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6822","nonce":2282009792},{"type":"v1","addr":"172.21.15.59:6823","nonce":2282009792}]},"public_addr":"172.21.15.59:6819/2282009792","cluster_addr":"172.21.15.59:6821/2282009792","heartbeat_back_addr":"172.21.15.59:6825/2282009792","heartbeat_front_addr":"172.21.15.59:6823/2282009792","state":["exists","up"]},{"osd":3,"uuid":"58994c22-3506-45b6-a449-81b31271ed94","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6826","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6827","nonce":4167347038}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6828","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6829","nonce":4167347038}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6832","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6833","nonce":4167347038}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6830","nonce":4167347038},{"type":"v1","addr":"172.21.15.59:6831","nonce":4167347038}]},"public_addr":"172.21.15.59:6827/4167347038","cluster_addr":"172.21.15.59:6829/4167347038","heartbeat_back_addr":"172.21.15.59:6833/4167347038","heartbeat_front_addr":"172.21.15.59:6831/4167347038","state":["exists","up"]},{"osd":4,"uuid":"193191de-111e-4026-a251-a4a1f9a7a8cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6801","nonce":304831184}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6803","nonce":304831184}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6807","nonce":304831184}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":304831184},{"type":"v1","addr":"172.21.15.64:6805","nonce":304831184}]},"public_addr":"172.21.15.64:6801/304831184","cluster_addr":"172.21.15.64:6803/304831184","heartbeat_back_addr":"172.21.15.64:6807/304831184","heartbeat_front_addr":"172.21.15.64:6805/304831184","state":["exists","up"]},{"osd":5,"uuid":"601697da-23df-4269-a707-eb9435930d2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6809","nonce":1151989321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6811","nonce":1151989321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6815","nonce":1151989321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":1151989321},{"type":"v1","addr":"172.21.15.64:6813","nonce":1151989321}]},"public_addr":"172.21.15.64:6809/1151989321","cluster_addr":"172.21.15.64:6811/1151989321","heartbeat_back_addr":"172.21.15.64:6815/1151989321","heartbeat_front_addr":"172.21.15.64:6813/1151989321","state":["exists","up"]},{"osd":6,"uuid":"02410d0f-0d83-403f-ba1c-eb45e7a474be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6817","nonce":2707336954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6819","nonce":2707336954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6823","nonce":2707336954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":2707336954},{"type":"v1","addr":"172.21.15.64:6821","nonce":2707336954}]},"public_addr":"172.21.15.64:6817/2707336954","cluster_addr":"172.21.15.64:6819/2707336954","heartbeat_back_addr":"172.21.15.64:6823/2707336954","heartbeat_front_addr":"172.21.15.64:6821/2707336954","state":["exists","up"]},{"osd":7,"uuid":"e21aa790-2584-4a0b-bbfc-340127422fe1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6825","nonce":2061142477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6827","nonce":2061142477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6831","nonce":2061142477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2061142477},{"type":"v1","addr":"172.21.15.64:6829","nonce":2061142477}]},"public_addr":"172.21.15.64:6825/2061142477","cluster_addr":"172.21.15.64:6827/2061142477","heartbeat_back_addr":"172.21.15.64:6831/2061142477","heartbeat_front_addr":"172.21.15.64:6829/2061142477","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:53:08.329520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:53:31.622804+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:53:56.343217+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:54:20.342834+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:54:41.812004+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:55:04.787063+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:55:27.741181+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T15:55:50.663035+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.59:6800/434932617":"2024-04-27T15:56:07.496595+0000","172.21.15.59:0/2069431914":"2024-04-27T15:56:07.496595+0000","172.21.15.59:0/3047449642":"2024-04-27T15:56:07.496595+0000","172.21.15.59:0/635818739":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/3932195089":"2024-04-27T15:56:07.496595+0000","172.21.15.59:6801/1212694920":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/1573688154":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/2655313575":"2024-04-27T15:56:07.496595+0000","172.21.15.59:0/1094121986":"2024-04-27T15:51:51.729465+0000","172.21.15.59:0/1707008100":"2024-04-27T15:51:29.308166+0000","172.21.15.59:6800/890201972":"2024-04-27T15:51:29.308166+0000","172.21.15.59:6800/1212694920":"2024-04-27T15:51:51.729465+0000","172.21.15.59:6801/890201972":"2024-04-27T15:51:29.308166+0000","172.21.15.59:6801/434932617":"2024-04-27T15:56:07.496595+0000","172.21.15.59:0/3060676991":"2024-04-27T15:51:29.308166+0000","172.21.15.59:0/2924841679":"2024-04-27T15:51:29.308166+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T15:56:32.186 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:32 smithi064 systemd[1]: Starting Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:56:32.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:32 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:32.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:32 smithi059 ceph-mon[26536]: Deploying daemon node-exporter.b on smithi064 2024-04-26T15:56:32.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:32 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3588507472' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:56:32.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:32 smithi059 ceph-mon[26536]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:32 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:32 smithi059 ceph-mon[21819]: Deploying daemon node-exporter.b on smithi064 2024-04-26T15:56:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:32 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3588507472' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:56:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:32 smithi059 ceph-mon[21819]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:32.586 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:32 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:32.586 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:32 smithi064 ceph-mon[22742]: Deploying daemon node-exporter.b on smithi064 2024-04-26T15:56:32.586 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:32 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3588507472' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T15:56:32.586 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:32 smithi064 ceph-mon[22742]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:32.586 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:32 smithi064 bash[38077]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-26T15:56:32.649 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T15:56:32.650 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T15:56:32.650 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T15:56:32.651 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T15:56:32.651 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T15:56:32.651 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T15:56:32.652 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T15:56:32.652 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T15:56:32.936 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:32 smithi064 bash[38077]: Getting image source signatures 2024-04-26T15:56:32.936 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:32 smithi064 bash[38077]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-26T15:56:32.936 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:32 smithi064 bash[38077]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-26T15:56:32.937 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:32 smithi064 bash[38077]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-26T15:56:33.059 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:33.062 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:33.063 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:33.065 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:33.074 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:33.074 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:33.128 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:33.128 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:34.186 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:33 smithi064 bash[38077]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-26T15:56:34.186 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 bash[38077]: Writing manifest to image destination 2024-04-26T15:56:34.604 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 podman[38077]: 2024-04-26 15:56:34.287819526 +0000 UTC m=+2.169378143 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-26T15:56:34.604 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 podman[38077]: 2024-04-26 15:56:34.369571175 +0000 UTC m=+2.251129778 container create 40b9dddd8b95f8dc6b80e7ab89d42d98a28bcb233abc690337965ec81cc0b2d2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T15:56:34.604 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 podman[38077]: 2024-04-26 15:56:34.538043292 +0000 UTC m=+2.419601897 container init 40b9dddd8b95f8dc6b80e7ab89d42d98a28bcb233abc690337965ec81cc0b2d2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T15:56:34.604 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 podman[38077]: 2024-04-26 15:56:34.541275056 +0000 UTC m=+2.422833659 container start 40b9dddd8b95f8dc6b80e7ab89d42d98a28bcb233abc690337965ec81cc0b2d2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T15:56:34.604 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.546Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-26T15:56:34.604 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.546Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-26T15:56:34.604 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-26T15:56:34.604 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T15:56:34.604 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-26T15:56:34.604 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=arp 2024-04-26T15:56:34.605 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-26T15:56:34.605 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-26T15:56:34.605 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-26T15:56:34.605 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-26T15:56:34.605 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-26T15:56:34.605 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-26T15:56:34.605 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-26T15:56:34.605 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=edac 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-26T15:56:34.606 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=os 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=stat 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-26T15:56:34.607 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=time 2024-04-26T15:56:34.608 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-26T15:56:34.608 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=uname 2024-04-26T15:56:34.608 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-26T15:56:34.608 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-26T15:56:34.608 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-26T15:56:34.608 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.547Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-26T15:56:34.608 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[38130]: ts=2024-04-26T15:56:34.548Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-26T15:56:34.608 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 bash[38077]: 40b9dddd8b95f8dc6b80e7ab89d42d98a28bcb233abc690337965ec81cc0b2d2 2024-04-26T15:56:34.936 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 15:56:34 smithi064 systemd[1]: Started Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:56:35.133 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:56:34] "GET /metrics HTTP/1.1" 200 191526 "" "Prometheus/2.33.4" 2024-04-26T15:56:35.133 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:35 smithi059 ceph-mon[21819]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:35.134 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:35 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:35.134 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:35 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:35.134 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:35 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:56:35.134 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:35 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:56:35.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:35 smithi059 ceph-mon[26536]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:35.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:35 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:35.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:35 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:35.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:35 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:56:35.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:35 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:56:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:35 smithi064 ceph-mon[22742]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:35 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:35 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:56:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:35 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:56:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:35 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:56:37.274 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:37 smithi059 ceph-mon[26536]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:37.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:37 smithi059 ceph-mon[21819]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:37.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:37 smithi064 ceph-mon[22742]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:38.457 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:38 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:38.458 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:38 smithi059 ceph-mon[26536]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:38.458 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:38 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:38.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:38 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:38.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:38 smithi059 ceph-mon[21819]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:38.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:38 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:38.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:38 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:38.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:38 smithi064 ceph-mon[22742]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:38 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:56:39.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:56:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:56:41.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:41 smithi059 ceph-mon[21819]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:41.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:41 smithi059 ceph-mon[26536]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:41.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:41 smithi064 ceph-mon[22742]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:43.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:43 smithi059 ceph-mon[21819]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:43.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:43 smithi059 ceph-mon[26536]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:43.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:43 smithi064 ceph-mon[22742]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:43.631 INFO:teuthology.orchestra.run.smithi059.stdout:55834574886 2024-04-26T15:56:43.631 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T15:56:44.969 INFO:teuthology.orchestra.run.smithi059.stdout:77309411363 2024-04-26T15:56:44.969 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T15:56:45.228 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:45 smithi059 ceph-mon[26536]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:45.229 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:45 smithi059 ceph-mon[21819]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:45.229 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:56:44] "GET /metrics HTTP/1.1" 200 191503 "" "Prometheus/2.33.4" 2024-04-26T15:56:45.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:45 smithi064 ceph-mon[22742]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:45.613 INFO:teuthology.orchestra.run.smithi059.stdout:236223201293 2024-04-26T15:56:45.613 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T15:56:46.005 INFO:teuthology.orchestra.run.smithi059.stdout:34359738412 2024-04-26T15:56:46.005 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T15:56:47.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:47 smithi064 ceph-mon[22742]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:47.443 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:47 smithi059 ceph-mon[21819]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:47.443 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:47 smithi059 ceph-mon[26536]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:47.741 INFO:teuthology.orchestra.run.smithi059.stdout:167503724564 2024-04-26T15:56:47.741 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T15:56:49.142 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:56:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:56:49.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:49 smithi059 ceph-mon[21819]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:49.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:49 smithi059 ceph-mon[26536]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:49.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:49 smithi064 ceph-mon[22742]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:49.585 INFO:teuthology.orchestra.run.smithi059.stdout:133143986202 2024-04-26T15:56:49.585 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T15:56:49.622 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:49.624 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:49.624 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:49.625 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:49.927 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:50.375 INFO:teuthology.orchestra.run.smithi059.stdout:201863462930 2024-04-26T15:56:50.375 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T15:56:50.584 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:56:51.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:51 smithi059 ceph-mon[21819]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:51.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:51 smithi059 ceph-mon[26536]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:51.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:51 smithi064 ceph-mon[22742]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:53.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:53 smithi059 ceph-mon[26536]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:53 smithi059 ceph-mon[21819]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:53.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:53 smithi064 ceph-mon[22742]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:55.249 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:56:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:56:54] "GET /metrics HTTP/1.1" 200 191491 "" "Prometheus/2.33.4" 2024-04-26T15:56:55.249 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:55 smithi059 ceph-mon[21819]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:55.250 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:55 smithi059 ceph-mon[26536]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:55.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:55 smithi064 ceph-mon[22742]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:55.554 INFO:teuthology.orchestra.run.smithi059.stdout:167503724565 2024-04-26T15:56:56.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:56 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/233740583' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T15:56:56.432 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:56 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/233740583' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T15:56:56.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:56 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/233740583' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T15:56:57.403 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:57 smithi059 ceph-mon[21819]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:57.404 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:57 smithi059 ceph-mon[26536]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:57.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:57 smithi064 ceph-mon[22742]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:57.701 INFO:teuthology.orchestra.run.smithi059.stdout:34359738415 2024-04-26T15:56:58.297 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:58 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2299784714' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T15:56:58.297 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:58 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2299784714' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T15:56:58.324 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724564 got 167503724565 for osd.5 2024-04-26T15:56:58.324 DEBUG:teuthology.parallel:result is None 2024-04-26T15:56:58.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:58 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2299784714' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T15:56:59.150 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:56:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:56:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:56:59.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:59 smithi059 ceph-mon[21819]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:59.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:56:59 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/4246270432' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T15:56:59.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:59 smithi059 ceph-mon[26536]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:59.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:56:59 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/4246270432' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T15:56:59.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:59 smithi064 ceph-mon[22742]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:56:59.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:56:59 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/4246270432' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T15:56:59.505 INFO:teuthology.orchestra.run.smithi059.stdout:111669149729 2024-04-26T15:56:59.506 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T15:56:59.870 INFO:teuthology.orchestra.run.smithi059.stdout:55834574890 2024-04-26T15:57:00.241 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:57:00.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:00 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3791973375' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T15:57:00.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:00 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3791973375' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T15:57:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:00 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3791973375' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T15:57:01.042 INFO:teuthology.orchestra.run.smithi059.stdout:77309411366 2024-04-26T15:57:01.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:01 smithi059 ceph-mon[21819]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:01.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:01 smithi059 ceph-mon[26536]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:01.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:01 smithi064 ceph-mon[22742]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:02.290 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574886 got 55834574890 for osd.1 2024-04-26T15:57:02.291 DEBUG:teuthology.parallel:result is None 2024-04-26T15:57:02.864 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738412 got 34359738415 for osd.0 2024-04-26T15:57:02.864 DEBUG:teuthology.parallel:result is None 2024-04-26T15:57:03.010 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411363 got 77309411366 for osd.2 2024-04-26T15:57:03.010 DEBUG:teuthology.parallel:result is None 2024-04-26T15:57:03.105 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:57:03.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:02 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:03.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:02 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:03.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:02 smithi064 ceph-mon[22742]: Deploying daemon alertmanager.a on smithi059 2024-04-26T15:57:03.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:02 smithi064 ceph-mon[22742]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:03.231 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:02 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:03.231 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:02 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:03.231 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:02 smithi059 ceph-mon[21819]: Deploying daemon alertmanager.a on smithi059 2024-04-26T15:57:03.231 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:02 smithi059 ceph-mon[21819]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:03.231 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:02 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:03.231 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:02 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:03.231 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:02 smithi059 ceph-mon[26536]: Deploying daemon alertmanager.a on smithi059 2024-04-26T15:57:03.231 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:02 smithi059 ceph-mon[26536]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:03.296 INFO:teuthology.orchestra.run.smithi059.stdout:236223201296 2024-04-26T15:57:04.324 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:04 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2595366199' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T15:57:04.325 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:04 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2595366199' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T15:57:04.354 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201293 got 236223201296 for osd.7 2024-04-26T15:57:04.354 DEBUG:teuthology.parallel:result is None 2024-04-26T15:57:04.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:04 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2595366199' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T15:57:05.250 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:05 smithi059 ceph-mon[21819]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:05.250 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:05 smithi059 ceph-mon[26536]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:05.251 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:57:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:57:04] "GET /metrics HTTP/1.1" 200 191491 "" "Prometheus/2.33.4" 2024-04-26T15:57:05.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:05 smithi064 ceph-mon[22742]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:06.214 INFO:teuthology.orchestra.run.smithi059.stdout:133143986206 2024-04-26T15:57:07.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:07 smithi064 ceph-mon[22742]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:07.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:07 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2741330615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T15:57:07.499 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:07 smithi059 ceph-mon[21819]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:07.499 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:07 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2741330615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T15:57:07.499 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:07 smithi059 ceph-mon[26536]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:07.499 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:07 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2741330615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T15:57:07.813 INFO:teuthology.orchestra.run.smithi059.stdout:111669149731 2024-04-26T15:57:08.434 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:08 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2364562974' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T15:57:08.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:08 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2364562974' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T15:57:08.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:08 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2364562974' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T15:57:09.146 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:57:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:57:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:57:09.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:09 smithi059 ceph-mon[26536]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:09.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:57:09.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:57:09.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:57:09.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:57:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:09 smithi059 ceph-mon[21819]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:57:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:57:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:57:09.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:57:09.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:09 smithi064 ceph-mon[22742]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:09.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:57:09.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:57:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:57:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:57:09.529 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986202 got 133143986206 for osd.4 2024-04-26T15:57:09.529 DEBUG:teuthology.parallel:result is None 2024-04-26T15:57:09.814 INFO:teuthology.orchestra.run.smithi059.stdout:201863462933 2024-04-26T15:57:09.940 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149729 got 111669149731 for osd.3 2024-04-26T15:57:09.940 DEBUG:teuthology.parallel:result is None 2024-04-26T15:57:10.196 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:10 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2041231712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T15:57:10.197 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:10 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2041231712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T15:57:10.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:10 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2041231712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T15:57:11.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:11 smithi059 ceph-mon[26536]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:11.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:11 smithi059 ceph-mon[21819]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:11.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:11 smithi064 ceph-mon[22742]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:11.630 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462930 got 201863462933 for osd.6 2024-04-26T15:57:11.631 DEBUG:teuthology.parallel:result is None 2024-04-26T15:57:11.631 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T15:57:11.631 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T15:57:13.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:13 smithi059 ceph-mon[21819]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:13.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:13 smithi059 ceph-mon[26536]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:13.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:13 smithi064 ceph-mon[22742]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:13.631 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:57:15.240 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:15 smithi059 ceph-mon[21819]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:15.241 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:57:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:57:14] "GET /metrics HTTP/1.1" 200 191491 "" "Prometheus/2.33.4" 2024-04-26T15:57:15.241 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:15 smithi059 ceph-mon[26536]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:15.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:15 smithi064 ceph-mon[22742]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:15.582 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 podman[44636]: 2024-04-26 15:57:15.4202537 +0000 UTC m=+0.427439630 container create e7720373f038f4915c016682b8601fcf9c11ac89522f0c1012968968e70b9704 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:15.882 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 podman[44636]: 2024-04-26 15:57:15.731120827 +0000 UTC m=+0.738306723 container init e7720373f038f4915c016682b8601fcf9c11ac89522f0c1012968968e70b9704 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:15.882 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 podman[44636]: 2024-04-26 15:57:15.73416907 +0000 UTC m=+0.741354957 container start e7720373f038f4915c016682b8601fcf9c11ac89522f0c1012968968e70b9704 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:15.882 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:15.745Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-26T15:57:15.882 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:15.746Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-26T15:57:15.882 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:15.747Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.59 port=9094 2024-04-26T15:57:15.882 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:15.748Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T15:57:15.882 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:15.774Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T15:57:15.882 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:15.774Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T15:57:15.883 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:15.777Z caller=main.go:518 msg=Listening address=:9093 2024-04-26T15:57:15.883 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:15.777Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-26T15:57:15.883 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 bash[44636]: e7720373f038f4915c016682b8601fcf9c11ac89522f0c1012968968e70b9704 2024-04-26T15:57:16.131 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:15 smithi059 systemd[1]: Started Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:57:16.327 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:57:16.328 INFO:teuthology.orchestra.run.smithi059.stderr:dumped all 2024-04-26T15:57:17.244 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.245 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.245 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.245 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:57:17.245 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:57:17.245 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.245 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[26536]: Deploying daemon grafana.a on smithi064 2024-04-26T15:57:17.245 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[26536]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:17.245 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.245 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.246 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.246 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:57:17.246 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:57:17.246 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.246 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[21819]: Deploying daemon grafana.a on smithi064 2024-04-26T15:57:17.246 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:16 smithi059 ceph-mon[21819]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:17.276 INFO:teuthology.orchestra.run.smithi059.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-04-26T15:57:16.088204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48976,"kb_used_data":6160,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912240,"statfs":{"total":767960285184,"available":767910133760,"internally_reserved":0,"allocated":6307840,"data_stored":3510994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":263,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-26T15:56:08.168091+0000","last_change":"2024-04-26T15:55:56.214974+0000","last_active":"2024-04-26T15:56:08.168091+0000","last_peered":"2024-04-26T15:56:08.168091+0000","last_clean":"2024-04-26T15:56:08.168091+0000","last_became_active":"2024-04-26T15:55:56.214530+0000","last_became_peered":"2024-04-26T15:55:56.214530+0000","last_unstale":"2024-04-26T15:56:08.168091+0000","last_undegraded":"2024-04-26T15:56:08.168091+0000","last_fullsized":"2024-04-26T15:56:08.168091+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T15:54:00.338927+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T15:54:00.338927+0000","last_clean_scrub_stamp":"2024-04-26T15:54:00.338927+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T19:48:44.756203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.35499999999999998}]},{"osd":1,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.581,"5min":1.581,"15min":1.581},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":19.684000000000001,"5min":19.684000000000001,"15min":19.684000000000001},"last":0.48499999999999999}]},{"osd":2,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.502,"5min":1.502,"15min":1.502},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":18.253,"5min":18.253,"15min":18.253},"last":0.52400000000000002}]},{"osd":3,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.5980000000000001,"5min":1.5980000000000001,"15min":1.5980000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":19.640000000000001,"5min":19.640000000000001,"15min":19.640000000000001},"last":0.56599999999999995}]},{"osd":4,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.51000000000000001}]},{"osd":5,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.58199999999999996}]},{"osd":6,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.59599999999999997}]}]},{"osd":6,"up_from":47,"seq":201863462935,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.40400000000000003}]},{"osd":1,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.438}]},{"osd":2,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.45600000000000002}]},{"osd":3,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.623}]},{"osd":4,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.69199999999999995}]},{"osd":5,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.64900000000000002}]},{"osd":7,"last update":"Fri Apr 26 15:56:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.67000000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:56:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.373,"15min":0.35199999999999998},"min":{"1min":0.214,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.629,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.374,"15min":0.35699999999999998},"min":{"1min":0.28799999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.55700000000000005,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.499}]},{"osd":2,"last update":"Fri Apr 26 15:57:08 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.41099999999999998,"15min":0.40699999999999997},"min":{"1min":0.26500000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.58199999999999996,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.40999999999999998,"15min":0.39600000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.61299999999999999,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.55100000000000005}]},{"osd":3,"last update":"Fri Apr 26 15:56:22 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.49199999999999999,"15min":0.497},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.434},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.51600000000000001}]},{"osd":4,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58199999999999996,"15min":0.57999999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.629,"5min":0.68100000000000005,"15min":0.68899999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.90000000000000002,"5min":2.8039999999999998,"15min":2.8039999999999998},"last":0.67800000000000005}]},{"osd":5,"last update":"Fri Apr 26 15:56:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.73299999999999998}]},{"osd":6,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.68999999999999995}]},{"osd":7,"last update":"Fri Apr 26 15:56:52 2024","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.625},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.75700000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 15:56:39 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.34300000000000003,"15min":0.32200000000000001},"min":{"1min":0.22500000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.52800000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.379,"15min":0.36299999999999999},"min":{"1min":0.254,"5min":0.216,"15min":0.216},"max":{"1min":0.60199999999999998,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.45800000000000002}]},{"osd":2,"last update":"Fri Apr 26 15:57:08 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.374,"5min":0.41099999999999998,"15min":0.41399999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.51200000000000001,"5min":0.625,"15min":0.625},"last":0.438}]},{"osd":3,"last update":"Fri Apr 26 15:56:29 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.47399999999999998,"15min":0.48399999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.58499999999999996,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.376},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.49199999999999999,"15min":0.504},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.56499999999999995,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.505}]},{"osd":4,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.58899999999999997,"15min":0.60299999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.58599999999999997,"15min":0.59199999999999997},"min":{"1min":0.38700000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.56599999999999995}]},{"osd":5,"last update":"Fri Apr 26 15:57:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.84499999999999997,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.626,"5min":0.59299999999999997,"15min":0.58699999999999997},"min":{"1min":0.438,"5min":0.379,"15min":0.379},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.60399999999999998}]},{"osd":6,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.71799999999999997}]},{"osd":7,"last update":"Fri Apr 26 15:56:57 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.628},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.75800000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:57:08 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.379,"15min":0.36699999999999999},"min":{"1min":0.26000000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.64100000000000001,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.38300000000000001,"15min":0.36899999999999999},"min":{"1min":0.28000000000000003,"5min":0.251,"15min":0.251},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.45200000000000001}]},{"osd":1,"last update":"Fri Apr 26 15:57:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.41199999999999998,"15min":0.39700000000000002},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.40100000000000002,"15min":0.39700000000000002},"min":{"1min":0.26100000000000001,"5min":0.248,"15min":0.248},"max":{"1min":0.56499999999999995,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.54100000000000004}]},{"osd":3,"last update":"Fri Apr 26 15:56:29 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.48099999999999998,"15min":0.48199999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.63600000000000001,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.48999999999999999,"15min":0.49399999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.58699999999999997}]},{"osd":4,"last update":"Fri Apr 26 15:56:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.59199999999999997,"15min":0.59699999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.747,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.59299999999999997,"15min":0.59799999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72199999999999998,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.92400000000000004}]},{"osd":5,"last update":"Fri Apr 26 15:57:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57899999999999996,"15min":0.57699999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.61099999999999999,"15min":0.60399999999999998},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.94099999999999995}]},{"osd":6,"last update":"Fri Apr 26 15:56:29 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.80300000000000005}]},{"osd":7,"last update":"Fri Apr 26 15:56:53 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.87}]}]},{"osd":3,"up_from":26,"seq":111669149733,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.55900000000000005,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.44800000000000001,"15min":0.45300000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.66900000000000004,"5min":0.753,"15min":0.753},"last":0.433}]},{"osd":1,"last update":"Fri Apr 26 15:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59799999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.374},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.48199999999999998,"15min":0.48599999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.67700000000000005,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.40699999999999997}]},{"osd":2,"last update":"Fri Apr 26 15:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.497,"15min":0.501},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.443},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.48399999999999999,"15min":0.48599999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63300000000000001,"5min":0.749,"15min":0.749},"last":0.39400000000000002}]},{"osd":4,"last update":"Fri Apr 26 15:56:47 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.40699999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.64100000000000001,"15min":0.64700000000000002},"min":{"1min":0.439,"5min":0.37,"15min":0.37},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.49299999999999999}]},{"osd":5,"last update":"Fri Apr 26 15:57:13 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.63100000000000001,"15min":0.627},"min":{"1min":0.41299999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.65900000000000003,"15min":0.66700000000000004},"min":{"1min":0.47299999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.85899999999999999,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.628}]},{"osd":6,"last update":"Fri Apr 26 15:56:31 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.54600000000000004}]},{"osd":7,"last update":"Fri Apr 26 15:56:53 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.51800000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986208,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.101,"15min":1.1930000000000001},"min":{"1min":0.35399999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.68300000000000005,"5min":12.826000000000001,"15min":12.826000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55000000000000004,"15min":0.55100000000000005},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.755,"5min":0.877,"15min":0.877},"last":0.61799999999999999}]},{"osd":1,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":1.415,"15min":1.5589999999999999},"min":{"1min":0.378,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.70599999999999996,"5min":19.629999999999999,"15min":19.629999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.401,"15min":1.5449999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.71599999999999997,"5min":18.413,"15min":18.413},"last":0.54600000000000004}]},{"osd":2,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.135,"15min":1.2290000000000001},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73499999999999999,"5min":12.853,"15min":12.853},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57499999999999996,"15min":0.57399999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72099999999999997,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.59299999999999997}]},{"osd":3,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.57199999999999995,"15min":0.56599999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76000000000000001,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.41899999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.75800000000000001,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.629}]},{"osd":5,"last update":"Fri Apr 26 15:57:12 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.435,"15min":0.42499999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.55200000000000005,"15min":0.53600000000000003},"min":{"1min":0.36499999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.65400000000000003}]},{"osd":6,"last update":"Fri Apr 26 15:56:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.433}]},{"osd":7,"last update":"Fri Apr 26 15:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.57599999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724570,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:57:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.58499999999999996,"15min":0.59299999999999997},"min":{"1min":0.41099999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.84499999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54400000000000004,"15min":0.54200000000000004},"min":{"1min":0.38400000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78500000000000003,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.71299999999999997}]},{"osd":1,"last update":"Fri Apr 26 15:57:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57099999999999995,"15min":0.56899999999999995},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69699999999999995,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.622},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.40200000000000002,"5min":0.375,"15min":0.375},"max":{"1min":0.70299999999999996,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.67100000000000004}]},{"osd":2,"last update":"Fri Apr 26 15:57:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55600000000000005,"15min":0.56000000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.83299999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.45900000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.75,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.72399999999999998}]},{"osd":3,"last update":"Fri Apr 26 15:57:10 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58499999999999996,"15min":0.58599999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77700000000000002,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.76000000000000001}]},{"osd":4,"last update":"Fri Apr 26 15:57:10 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.44900000000000001,"15min":0.439},"min":{"1min":0.35799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.505},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.45600000000000002,"15min":0.44400000000000001},"min":{"1min":0.33700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.41699999999999998}]},{"osd":6,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.432},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.45000000000000001}]},{"osd":7,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.47099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T15:57:17.277 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T15:57:17.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:16 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:16 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:16 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:16 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:57:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:16 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T15:57:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:16 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:16 smithi064 ceph-mon[22742]: Deploying daemon grafana.a on smithi064 2024-04-26T15:57:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:16 smithi064 ceph-mon[22742]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:17.506 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:57:17.923 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:17.749Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000931985s 2024-04-26T15:57:18.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:18 smithi059 ceph-mon[26536]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:57:18.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:18 smithi059 ceph-mon[21819]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:57:18.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:18 smithi064 ceph-mon[22742]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:57:19.135 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:57:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:57:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:57:19.229 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:57:19.230 INFO:teuthology.orchestra.run.smithi059.stderr:dumped all 2024-04-26T15:57:19.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:19 smithi064 ceph-mon[22742]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:19.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:19 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:19.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:19 smithi059 ceph-mon[26536]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:19.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:19 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:19.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:19 smithi059 ceph-mon[21819]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:19.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:19 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:19.891 INFO:teuthology.orchestra.run.smithi059.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-04-26T15:57:18.088516+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48976,"kb_used_data":6160,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912240,"statfs":{"total":767960285184,"available":767910133760,"internally_reserved":0,"allocated":6307840,"data_stored":3510994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002028"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":263,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-26T15:56:08.168091+0000","last_change":"2024-04-26T15:55:56.214974+0000","last_active":"2024-04-26T15:56:08.168091+0000","last_peered":"2024-04-26T15:56:08.168091+0000","last_clean":"2024-04-26T15:56:08.168091+0000","last_became_active":"2024-04-26T15:55:56.214530+0000","last_became_peered":"2024-04-26T15:55:56.214530+0000","last_unstale":"2024-04-26T15:56:08.168091+0000","last_undegraded":"2024-04-26T15:56:08.168091+0000","last_fullsized":"2024-04-26T15:56:08.168091+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T15:54:00.338927+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T15:54:00.338927+0000","last_clean_scrub_stamp":"2024-04-26T15:54:00.338927+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T19:48:44.756203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.35499999999999998}]},{"osd":1,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.581,"5min":1.581,"15min":1.581},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":19.684000000000001,"5min":19.684000000000001,"15min":19.684000000000001},"last":0.48499999999999999}]},{"osd":2,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.502,"5min":1.502,"15min":1.502},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":18.253,"5min":18.253,"15min":18.253},"last":0.52400000000000002}]},{"osd":3,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.5980000000000001,"5min":1.5980000000000001,"15min":1.5980000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":19.640000000000001,"5min":19.640000000000001,"15min":19.640000000000001},"last":0.56599999999999995}]},{"osd":4,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.51000000000000001}]},{"osd":5,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.58199999999999996}]},{"osd":6,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.59599999999999997}]}]},{"osd":6,"up_from":47,"seq":201863462935,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.40400000000000003}]},{"osd":1,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.438}]},{"osd":2,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.45600000000000002}]},{"osd":3,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.623}]},{"osd":4,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.69199999999999995}]},{"osd":5,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.64900000000000002}]},{"osd":7,"last update":"Fri Apr 26 15:56:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.67000000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:56:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.373,"15min":0.35199999999999998},"min":{"1min":0.214,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.629,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.374,"15min":0.35699999999999998},"min":{"1min":0.28799999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.55700000000000005,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.499}]},{"osd":2,"last update":"Fri Apr 26 15:57:08 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.41099999999999998,"15min":0.40699999999999997},"min":{"1min":0.26500000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.58199999999999996,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.40999999999999998,"15min":0.39600000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.61299999999999999,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.55100000000000005}]},{"osd":3,"last update":"Fri Apr 26 15:56:22 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.49199999999999999,"15min":0.497},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.434},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.51600000000000001}]},{"osd":4,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58199999999999996,"15min":0.57999999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.629,"5min":0.68100000000000005,"15min":0.68899999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.90000000000000002,"5min":2.8039999999999998,"15min":2.8039999999999998},"last":0.67800000000000005}]},{"osd":5,"last update":"Fri Apr 26 15:56:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.73299999999999998}]},{"osd":6,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.68999999999999995}]},{"osd":7,"last update":"Fri Apr 26 15:56:52 2024","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.625},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.75700000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 15:56:39 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.34300000000000003,"15min":0.32200000000000001},"min":{"1min":0.22500000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.52800000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.379,"15min":0.36299999999999999},"min":{"1min":0.254,"5min":0.216,"15min":0.216},"max":{"1min":0.60199999999999998,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.45800000000000002}]},{"osd":2,"last update":"Fri Apr 26 15:57:08 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.374,"5min":0.41099999999999998,"15min":0.41399999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.51200000000000001,"5min":0.625,"15min":0.625},"last":0.438}]},{"osd":3,"last update":"Fri Apr 26 15:56:29 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.47399999999999998,"15min":0.48399999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.58499999999999996,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.376},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.49199999999999999,"15min":0.504},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.56499999999999995,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.505}]},{"osd":4,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.58899999999999997,"15min":0.60299999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.58599999999999997,"15min":0.59199999999999997},"min":{"1min":0.38700000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.56599999999999995}]},{"osd":5,"last update":"Fri Apr 26 15:57:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.84499999999999997,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.626,"5min":0.59299999999999997,"15min":0.58699999999999997},"min":{"1min":0.438,"5min":0.379,"15min":0.379},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.60399999999999998}]},{"osd":6,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.71799999999999997}]},{"osd":7,"last update":"Fri Apr 26 15:56:57 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.628},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.75800000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:57:08 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.379,"15min":0.36699999999999999},"min":{"1min":0.26000000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.64100000000000001,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.38300000000000001,"15min":0.36899999999999999},"min":{"1min":0.28000000000000003,"5min":0.251,"15min":0.251},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.45200000000000001}]},{"osd":1,"last update":"Fri Apr 26 15:57:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.41199999999999998,"15min":0.39700000000000002},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.40100000000000002,"15min":0.39700000000000002},"min":{"1min":0.26100000000000001,"5min":0.248,"15min":0.248},"max":{"1min":0.56499999999999995,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.54100000000000004}]},{"osd":3,"last update":"Fri Apr 26 15:56:29 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.48099999999999998,"15min":0.48199999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.63600000000000001,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.48999999999999999,"15min":0.49399999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.58699999999999997}]},{"osd":4,"last update":"Fri Apr 26 15:56:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.59199999999999997,"15min":0.59699999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.747,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.59299999999999997,"15min":0.59799999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72199999999999998,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.92400000000000004}]},{"osd":5,"last update":"Fri Apr 26 15:57:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57899999999999996,"15min":0.57699999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.61099999999999999,"15min":0.60399999999999998},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.94099999999999995}]},{"osd":6,"last update":"Fri Apr 26 15:56:29 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.80300000000000005}]},{"osd":7,"last update":"Fri Apr 26 15:56:53 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.87}]}]},{"osd":3,"up_from":26,"seq":111669149733,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.55900000000000005,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.44800000000000001,"15min":0.45300000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.66900000000000004,"5min":0.753,"15min":0.753},"last":0.433}]},{"osd":1,"last update":"Fri Apr 26 15:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59799999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.374},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.48199999999999998,"15min":0.48599999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.67700000000000005,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.40699999999999997}]},{"osd":2,"last update":"Fri Apr 26 15:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.497,"15min":0.501},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.443},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.48399999999999999,"15min":0.48599999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63300000000000001,"5min":0.749,"15min":0.749},"last":0.39400000000000002}]},{"osd":4,"last update":"Fri Apr 26 15:56:47 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.40699999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.64100000000000001,"15min":0.64700000000000002},"min":{"1min":0.439,"5min":0.37,"15min":0.37},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.49299999999999999}]},{"osd":5,"last update":"Fri Apr 26 15:57:13 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.63100000000000001,"15min":0.627},"min":{"1min":0.41299999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.65900000000000003,"15min":0.66700000000000004},"min":{"1min":0.47299999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.85899999999999999,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.628}]},{"osd":6,"last update":"Fri Apr 26 15:56:31 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.54600000000000004}]},{"osd":7,"last update":"Fri Apr 26 15:56:53 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.51800000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986208,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.101,"15min":1.1930000000000001},"min":{"1min":0.35399999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.68300000000000005,"5min":12.826000000000001,"15min":12.826000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55000000000000004,"15min":0.55100000000000005},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.755,"5min":0.877,"15min":0.877},"last":0.61799999999999999}]},{"osd":1,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":1.415,"15min":1.5589999999999999},"min":{"1min":0.378,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.70599999999999996,"5min":19.629999999999999,"15min":19.629999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.401,"15min":1.5449999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.71599999999999997,"5min":18.413,"15min":18.413},"last":0.54600000000000004}]},{"osd":2,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.135,"15min":1.2290000000000001},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73499999999999999,"5min":12.853,"15min":12.853},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57499999999999996,"15min":0.57399999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72099999999999997,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.59299999999999997}]},{"osd":3,"last update":"Fri Apr 26 15:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.57199999999999995,"15min":0.56599999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76000000000000001,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.41899999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.75800000000000001,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.629}]},{"osd":5,"last update":"Fri Apr 26 15:57:12 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.435,"15min":0.42499999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.55200000000000005,"15min":0.53600000000000003},"min":{"1min":0.36499999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.65400000000000003}]},{"osd":6,"last update":"Fri Apr 26 15:56:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.433}]},{"osd":7,"last update":"Fri Apr 26 15:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.57599999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724570,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 15:57:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.58499999999999996,"15min":0.59299999999999997},"min":{"1min":0.41099999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.84499999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54400000000000004,"15min":0.54200000000000004},"min":{"1min":0.38400000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78500000000000003,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.71299999999999997}]},{"osd":1,"last update":"Fri Apr 26 15:57:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57099999999999995,"15min":0.56899999999999995},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69699999999999995,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.622},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.40200000000000002,"5min":0.375,"15min":0.375},"max":{"1min":0.70299999999999996,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.67100000000000004}]},{"osd":2,"last update":"Fri Apr 26 15:57:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55600000000000005,"15min":0.56000000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.83299999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.45900000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.75,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.72399999999999998}]},{"osd":3,"last update":"Fri Apr 26 15:57:10 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58499999999999996,"15min":0.58599999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77700000000000002,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.76000000000000001}]},{"osd":4,"last update":"Fri Apr 26 15:57:10 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.44900000000000001,"15min":0.439},"min":{"1min":0.35799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.505},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.45600000000000002,"15min":0.44400000000000001},"min":{"1min":0.33700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.41699999999999998}]},{"osd":6,"last update":"Fri Apr 26 15:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.432},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.45000000000000001}]},{"osd":7,"last update":"Fri Apr 26 15:56:54 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.47099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T15:57:19.892 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T15:57:19.892 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T15:57:19.893 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T15:57:19.893 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T15:57:20.113 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T15:57:20.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:20 smithi064 ceph-mon[22742]: from='client.24526 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:57:20.461 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:20 smithi059 ceph-mon[21819]: from='client.24526 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:57:20.461 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:20 smithi059 ceph-mon[26536]: from='client.24526 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T15:57:21.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:21 smithi064 ceph-mon[22742]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:21.459 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:21 smithi059 ceph-mon[26536]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:21.459 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:21 smithi059 ceph-mon[21819]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:21.955 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T15:57:21.957 INFO:teuthology.orchestra.run.smithi059.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T15:57:22.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:22 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1844523990' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:22 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1844523990' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T15:57:22.505 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:22 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1844523990' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T15:57:22.531 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T15:57:22.532 INFO:tasks.cephadm:Setup complete, yielding 2024-04-26T15:57:22.532 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T15:57:22.550 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi059.front.sepia.ceph.com 2024-04-26T15:57:22.550 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-04-26T15:57:23.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:23 smithi064 ceph-mon[22742]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:23.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:23 smithi059 ceph-mon[26536]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:23.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:23 smithi059 ceph-mon[21819]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:24.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:24 smithi059 ceph-mon[26536]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:24.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:24 smithi059 ceph-mon[21819]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:24.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:24 smithi064 ceph-mon[22742]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:25.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:57:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:57:24] "GET /metrics HTTP/1.1" 200 191498 "" "Prometheus/2.33.4" 2024-04-26T15:57:25.618 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T15:57:25.618 INFO:teuthology.orchestra.run.smithi059.stdout: "id": "5f31e43f-ac1c-48e4-a45d-d4f4ca882b4a", 2024-04-26T15:57:25.618 INFO:teuthology.orchestra.run.smithi059.stdout: "name": "r", 2024-04-26T15:57:25.618 INFO:teuthology.orchestra.run.smithi059.stdout: "current_period": "9a47d632-4fe3-4e2b-80b6-db81506faef6", 2024-04-26T15:57:25.618 INFO:teuthology.orchestra.run.smithi059.stdout: "epoch": 1 2024-04-26T15:57:25.618 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T15:57:25.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:25 smithi064 ceph-mon[22742]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T15:57:25.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:25 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1979748088' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-26T15:57:25.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:25 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1979748088' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-26T15:57:25.942 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:25.752Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003708196s 2024-04-26T15:57:25.943 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:25 smithi059 ceph-mon[21819]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T15:57:25.943 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:25 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1979748088' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-26T15:57:25.943 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:25 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1979748088' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-26T15:57:25.943 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:25 smithi059 ceph-mon[26536]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T15:57:25.943 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:25 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1979748088' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-26T15:57:25.943 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:25 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1979748088' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-26T15:57:26.281 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-04-26T15:57:26.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:26 smithi064 ceph-mon[22742]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T15:57:26.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:26 smithi064 ceph-mon[22742]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:26.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:26 smithi064 ceph-mon[22742]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T15:57:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:26 smithi059 ceph-mon[26536]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T15:57:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:26 smithi059 ceph-mon[26536]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:26 smithi059 ceph-mon[26536]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T15:57:26.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:26 smithi059 ceph-mon[21819]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T15:57:26.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:26 smithi059 ceph-mon[21819]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:26.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:26 smithi059 ceph-mon[21819]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T15:57:28.145 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T15:57:28.145 INFO:teuthology.orchestra.run.smithi059.stdout: "id": "94a0e3d3-e849-4a13-a6e1-1c1b4ba527e7", 2024-04-26T15:57:28.145 INFO:teuthology.orchestra.run.smithi059.stdout: "name": "default", 2024-04-26T15:57:28.145 INFO:teuthology.orchestra.run.smithi059.stdout: "api_name": "default", 2024-04-26T15:57:28.145 INFO:teuthology.orchestra.run.smithi059.stdout: "is_master": "true", 2024-04-26T15:57:28.145 INFO:teuthology.orchestra.run.smithi059.stdout: "endpoints": [], 2024-04-26T15:57:28.145 INFO:teuthology.orchestra.run.smithi059.stdout: "hostnames": [], 2024-04-26T15:57:28.145 INFO:teuthology.orchestra.run.smithi059.stdout: "hostnames_s3website": [], 2024-04-26T15:57:28.146 INFO:teuthology.orchestra.run.smithi059.stdout: "master_zone": "", 2024-04-26T15:57:28.146 INFO:teuthology.orchestra.run.smithi059.stdout: "zones": [], 2024-04-26T15:57:28.146 INFO:teuthology.orchestra.run.smithi059.stdout: "placement_targets": [], 2024-04-26T15:57:28.146 INFO:teuthology.orchestra.run.smithi059.stdout: "default_placement": "", 2024-04-26T15:57:28.146 INFO:teuthology.orchestra.run.smithi059.stdout: "realm_id": "5f31e43f-ac1c-48e4-a45d-d4f4ca882b4a", 2024-04-26T15:57:28.146 INFO:teuthology.orchestra.run.smithi059.stdout: "sync_policy": { 2024-04-26T15:57:28.146 INFO:teuthology.orchestra.run.smithi059.stdout: "groups": [] 2024-04-26T15:57:28.146 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T15:57:28.146 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T15:57:28.731 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-04-26T15:57:29.101 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:57:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:57:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:57:29.102 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:28 smithi064 systemd[1]: Starting Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:57:29.296 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:29 smithi059 ceph-mon[26536]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:29.297 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:29 smithi059 ceph-mon[21819]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:29.353 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-mon[22742]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:29.354 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 podman[38810]: 2024-04-26 15:57:29.005284311 +0000 UTC m=+0.017634699 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-26T15:57:29.354 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 podman[38810]: 2024-04-26 15:57:29.11241238 +0000 UTC m=+0.124762775 container create 7a86c62f40c26f5c2353115abbf7c5d62bb6147e1a92abfab6eab65a0e9caff8 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, release=236.1648460182, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.tags=base rhel8) 2024-04-26T15:57:29.354 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 podman[38810]: 2024-04-26 15:57:29.306412618 +0000 UTC m=+0.318763006 container init 7a86c62f40c26f5c2353115abbf7c5d62bb6147e1a92abfab6eab65a0e9caff8 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, version=8.5, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, name=ubi8, io.buildah.version=1.24.2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.openshift.expose-services=, io.openshift.tags=base rhel8) 2024-04-26T15:57:29.354 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 podman[38810]: 2024-04-26 15:57:29.309059353 +0000 UTC m=+0.321409741 container start 7a86c62f40c26f5c2353115abbf7c5d62bb6147e1a92abfab6eab65a0e9caff8 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, description=Ceph Grafana Container, name=ubi8, io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.buildah.version=1.24.2) 2024-04-26T15:57:29.354 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-26T15:57:29.354 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-26T15:57:29.354 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="App mode production" logger=settings 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-26T15:57:29.355 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-26T15:57:29.356 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-04-26T15:57:29.356 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-04-26T15:57:29.356 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-04-26T15:57:29.356 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-04-26T15:57:29.356 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-04-26T15:57:29.356 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-04-26T15:57:29.356 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-04-26T15:57:29.356 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-04-26T15:57:29.356 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-04-26T15:57:29.356 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-04-26T15:57:29.357 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-04-26T15:57:29.604 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-04-26T15:57:29.604 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-04-26T15:57:29.605 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-04-26T15:57:29.606 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-04-26T15:57:29.606 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-04-26T15:57:29.606 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-04-26T15:57:29.606 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-04-26T15:57:29.606 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-04-26T15:57:29.606 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-04-26T15:57:29.607 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-04-26T15:57:29.607 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-04-26T15:57:29.607 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-04-26T15:57:29.607 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-04-26T15:57:29.607 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-04-26T15:57:29.607 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-04-26T15:57:29.607 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-04-26T15:57:29.608 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-04-26T15:57:29.609 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 bash[38810]: 7a86c62f40c26f5c2353115abbf7c5d62bb6147e1a92abfab6eab65a0e9caff8 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 systemd[1]: Started Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-26T15:57:29.610 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-04-26T15:57:29.611 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-04-26T15:57:29.612 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-04-26T15:57:29.612 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-04-26T15:57:29.612 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-04-26T15:57:29.612 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-04-26T15:57:29.613 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-04-26T15:57:29.613 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-04-26T15:57:29.613 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-26T15:57:29.613 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-04-26T15:57:29.613 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-04-26T15:57:29.613 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-04-26T15:57:29.613 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-04-26T15:57:29.613 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-04-26T15:57:29.614 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-04-26T15:57:29.614 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-04-26T15:57:29.614 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-04-26T15:57:29.614 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-04-26T15:57:29.614 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-04-26T15:57:29.614 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-04-26T15:57:29.614 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-04-26T15:57:29.614 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-04-26T15:57:29.614 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-04-26T15:57:29.615 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-26T15:57:29.616 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-04-26T15:57:29.617 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-26T15:57:29.618 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-04-26T15:57:29.619 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-26T15:57:29.620 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-04-26T15:57:29.620 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-04-26T15:57:29.855 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-04-26T15:57:29.855 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-04-26T15:57:29.855 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-04-26T15:57:29.855 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-04-26T15:57:29.855 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-04-26T15:57:29.855 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-04-26T15:57:29.856 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-26T15:57:29.857 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-26T15:57:29.857 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-26T15:57:29.857 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-04-26T15:57:29.857 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-26T15:57:29.857 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-26T15:57:29.858 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-04-26T15:57:29.859 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-04-26T15:57:29.860 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-04-26T15:57:29.861 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-04-26T15:57:29.861 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-04-26T15:57:29.861 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-04-26T15:57:29.861 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-04-26T15:57:29.861 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-04-26T15:57:29.861 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-04-26T15:57:29.861 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-26T15:57:29.861 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-04-26T15:57:29.861 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-04-26T15:57:29.861 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-04-26T15:57:29.862 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-04-26T15:57:29.862 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-04-26T15:57:29.862 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-26T15:57:29.862 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-04-26T15:57:29.862 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-04-26T15:57:29.863 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-04-26T15:57:29.864 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-04-26T15:57:29.865 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-26T15:57:29.866 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-26T15:57:29.867 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-04-26T15:57:29.868 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-04-26T15:57:29.869 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-04-26T15:57:29.870 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-04-26T15:57:29.871 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-04-26T15:57:29.871 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-04-26T15:57:29.871 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-04-26T15:57:29.871 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-04-26T15:57:29.871 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-04-26T15:57:29.871 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-04-26T15:57:30.105 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=537.231416ms 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-26T15:57:30.106 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-26T15:57:30.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-26T15:57:30.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-26T15:57:30.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-26T15:57:30.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-26T15:57:30.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-26T15:57:30.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:29+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-26T15:57:30.107 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 15:57:30 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T15:57:30+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-26T15:57:30.511 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T15:57:30.511 INFO:teuthology.orchestra.run.smithi059.stdout: "id": "c77933b7-fe75-4887-98df-c99f0c3dc1aa", 2024-04-26T15:57:30.511 INFO:teuthology.orchestra.run.smithi059.stdout: "name": "z", 2024-04-26T15:57:30.511 INFO:teuthology.orchestra.run.smithi059.stdout: "domain_root": "z.rgw.meta:root", 2024-04-26T15:57:30.511 INFO:teuthology.orchestra.run.smithi059.stdout: "control_pool": "z.rgw.control", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "gc_pool": "z.rgw.log:gc", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "lc_pool": "z.rgw.log:lc", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "log_pool": "z.rgw.log", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "roles_pool": "z.rgw.meta:roles", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "otp_pool": "z.rgw.otp", 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "system_key": { 2024-04-26T15:57:30.512 INFO:teuthology.orchestra.run.smithi059.stdout: "access_key": "", 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: "secret_key": "" 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: "placement_pools": [ 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: { 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: "key": "default-placement", 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: "val": { 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: "index_pool": "z.rgw.buckets.index", 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: "storage_classes": { 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: "STANDARD": { 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: "data_pool": "z.rgw.buckets.data" 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: "index_type": 0 2024-04-26T15:57:30.513 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T15:57:30.514 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T15:57:30.514 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T15:57:30.514 INFO:teuthology.orchestra.run.smithi059.stdout: "realm_id": "5f31e43f-ac1c-48e4-a45d-d4f4ca882b4a", 2024-04-26T15:57:30.514 INFO:teuthology.orchestra.run.smithi059.stdout: "notif_pool": "z.rgw.log:notif" 2024-04-26T15:57:30.514 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T15:57:30.809 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:30 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:30.809 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:30 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:57:30.809 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:30 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:30.809 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:30 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:57:30.809 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:30 smithi064 ceph-mon[22742]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-26T15:57:30.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:30 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:30.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:30 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:57:30.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:30 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:30.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:30 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:57:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:30 smithi059 ceph-mon[26536]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-26T15:57:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:30 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:30 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:57:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:30 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:30 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:57:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:30 smithi059 ceph-mon[21819]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-26T15:57:31.137 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-04-26T15:57:33.326 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:33 smithi059 ceph-mon[21819]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 545 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-04-26T15:57:33.326 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:33 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:33.326 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:33 smithi059 ceph-mon[26536]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 545 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-04-26T15:57:33.326 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:33 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:33 smithi064 ceph-mon[22742]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 545 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-04-26T15:57:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:33 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:34.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:34 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:34 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:34.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:34 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:35.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:57:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:57:34] "GET /metrics HTTP/1.1" 200 191498 "" "Prometheus/2.33.4" 2024-04-26T15:57:35.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:35 smithi059 ceph-mon[26536]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-04-26T15:57:35.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:35 smithi059 ceph-mon[26536]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T15:57:35.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:35 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1501695405' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-26T15:57:35.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:35 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-26T15:57:35.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:35 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:35.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:35 smithi059 ceph-mon[21819]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-04-26T15:57:35.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:35 smithi059 ceph-mon[21819]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T15:57:35.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:35 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1501695405' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-26T15:57:35.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:35 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-26T15:57:35.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:35 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:35 smithi064 ceph-mon[22742]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-04-26T15:57:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:35 smithi064 ceph-mon[22742]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T15:57:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:35 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1501695405' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-26T15:57:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:35 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-26T15:57:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:35 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:36.274 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:36 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-26T15:57:36.274 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:36 smithi059 ceph-mon[26536]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T15:57:36.274 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:36 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:36.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:36 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-26T15:57:36.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:36 smithi059 ceph-mon[21819]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T15:57:36.275 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:36 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:36.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:36 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-26T15:57:36.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:36 smithi064 ceph-mon[22742]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T15:57:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:36 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:37.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[21819]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T15:57:37.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[21819]: Reconfiguring daemon alertmanager.a on smithi059 2024-04-26T15:57:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[21819]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-26T15:57:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[21819]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T15:57:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1501695405' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-26T15:57:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-26T15:57:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-26T15:57:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[21819]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T15:57:37.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[26536]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T15:57:37.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[26536]: Reconfiguring daemon alertmanager.a on smithi059 2024-04-26T15:57:37.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[26536]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-26T15:57:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[26536]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T15:57:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1501695405' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-26T15:57:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-26T15:57:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-26T15:57:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:37 smithi059 ceph-mon[26536]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T15:57:37.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:37 smithi064 ceph-mon[22742]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T15:57:37.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:37 smithi064 ceph-mon[22742]: Reconfiguring daemon alertmanager.a on smithi059 2024-04-26T15:57:37.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:37 smithi064 ceph-mon[22742]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-26T15:57:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:37 smithi064 ceph-mon[22742]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T15:57:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:37 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1501695405' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-26T15:57:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:37 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-26T15:57:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:37 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-26T15:57:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:37 smithi064 ceph-mon[22742]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T15:57:38.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:38 smithi059 systemd[1]: Stopping Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:57:38.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:38 smithi059 bash[46271]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-26T15:57:39.153 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[44662]: level=info ts=2024-04-26T15:57:38.837Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-26T15:57:39.154 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:38 smithi059 podman[46284]: 2024-04-26 15:57:38.858221196 +0000 UTC m=+0.117223507 container died e7720373f038f4915c016682b8601fcf9c11ac89522f0c1012968968e70b9704 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:39.154 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:57:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:57:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:57:39.408 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:39 smithi059 ceph-mon[21819]: pgmap v55: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:39.408 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:39 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:39.409 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:39 smithi059 ceph-mon[21819]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T15:57:39.409 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:39 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-26T15:57:39.409 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:39 smithi059 ceph-mon[26536]: pgmap v55: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:39.409 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:39 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:39.409 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:39 smithi059 ceph-mon[26536]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T15:57:39.409 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:39 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-26T15:57:39.409 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:39 smithi059 podman[46284]: 2024-04-26 15:57:39.321605074 +0000 UTC m=+0.580607385 container cleanup e7720373f038f4915c016682b8601fcf9c11ac89522f0c1012968968e70b9704 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:39.409 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:39 smithi059 bash[46284]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T15:57:39.409 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:39 smithi059 bash[46313]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-26T15:57:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:39 smithi064 ceph-mon[22742]: pgmap v55: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:39 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:39 smithi064 ceph-mon[22742]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T15:57:39.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:39 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-26T15:57:39.713 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:39 smithi059 podman[46298]: 2024-04-26 15:57:39.631296644 +0000 UTC m=+0.767003283 container remove e7720373f038f4915c016682b8601fcf9c11ac89522f0c1012968968e70b9704 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:39.714 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:39 smithi059 podman[46298]: 2024-04-26 15:57:39.71511087 +0000 UTC m=+0.850817506 volume remove bd471ea15ddb84b6d37106b335553edf56b75e0508a8a884fdcb9baafc2e85be 2024-04-26T15:57:39.965 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:39 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T15:57:39.965 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:39 smithi059 systemd[1]: Stopped Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:57:39.965 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:39 smithi059 systemd[1]: Starting Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:57:39.965 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:39 smithi059 podman[46377]: 2024-04-26 15:57:39.860423997 +0000 UTC m=+0.019497191 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-26T15:57:39.965 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:39 smithi059 podman[46377]: 2024-04-26 15:57:39.966080633 +0000 UTC m=+0.125153864 volume create 606fff395b0ea7e9ad451673ef8507b09fa00ca125117dac36ac76a091003b15 2024-04-26T15:57:40.230 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-26T15:57:40.230 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-mon[26536]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T15:57:40.230 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-26T15:57:40.230 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 podman[46377]: 2024-04-26 15:57:40.049756664 +0000 UTC m=+0.208829853 container create 3d1458365b227c2c68e8f9e71085bb141a50e0075a8ad993dedb54ba2ea2918a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:40.231 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-26T15:57:40.231 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-mon[21819]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T15:57:40.231 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-26T15:57:40.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:40 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-26T15:57:40.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:40 smithi064 ceph-mon[22742]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T15:57:40.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:40 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-26T15:57:40.582 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 podman[46377]: 2024-04-26 15:57:40.319332405 +0000 UTC m=+0.478405597 container init 3d1458365b227c2c68e8f9e71085bb141a50e0075a8ad993dedb54ba2ea2918a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:40.582 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 podman[46377]: 2024-04-26 15:57:40.322054802 +0000 UTC m=+0.481127999 container start 3d1458365b227c2c68e8f9e71085bb141a50e0075a8ad993dedb54ba2ea2918a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:40.582 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T15:57:40.333Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-26T15:57:40.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T15:57:40.333Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-26T15:57:40.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T15:57:40.335Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.59 port=9094 2024-04-26T15:57:40.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T15:57:40.336Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T15:57:40.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T15:57:40.361Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T15:57:40.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T15:57:40.362Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T15:57:40.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T15:57:40.364Z caller=main.go:518 msg=Listening address=:9093 2024-04-26T15:57:40.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T15:57:40.364Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-26T15:57:40.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 bash[46377]: 3d1458365b227c2c68e8f9e71085bb141a50e0075a8ad993dedb54ba2ea2918a 2024-04-26T15:57:40.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:40 smithi059 systemd[1]: Started Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:57:41.258 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T15:57:41.258 INFO:teuthology.orchestra.run.smithi059.stdout: "id": "d21312eb-c5b3-4991-9c40-de2582235305", 2024-04-26T15:57:41.258 INFO:teuthology.orchestra.run.smithi059.stdout: "epoch": 1, 2024-04-26T15:57:41.258 INFO:teuthology.orchestra.run.smithi059.stdout: "predecessor_uuid": "9a47d632-4fe3-4e2b-80b6-db81506faef6", 2024-04-26T15:57:41.258 INFO:teuthology.orchestra.run.smithi059.stdout: "sync_status": [], 2024-04-26T15:57:41.258 INFO:teuthology.orchestra.run.smithi059.stdout: "period_map": { 2024-04-26T15:57:41.258 INFO:teuthology.orchestra.run.smithi059.stdout: "id": "d21312eb-c5b3-4991-9c40-de2582235305", 2024-04-26T15:57:41.258 INFO:teuthology.orchestra.run.smithi059.stdout: "zonegroups": [ 2024-04-26T15:57:41.258 INFO:teuthology.orchestra.run.smithi059.stdout: { 2024-04-26T15:57:41.258 INFO:teuthology.orchestra.run.smithi059.stdout: "id": "94a0e3d3-e849-4a13-a6e1-1c1b4ba527e7", 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "name": "default", 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "api_name": "default", 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "is_master": "true", 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "endpoints": [], 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "hostnames": [], 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "hostnames_s3website": [], 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "master_zone": "c77933b7-fe75-4887-98df-c99f0c3dc1aa", 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "zones": [ 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: { 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "id": "c77933b7-fe75-4887-98df-c99f0c3dc1aa", 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "name": "z", 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "endpoints": [], 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "log_meta": "false", 2024-04-26T15:57:41.259 INFO:teuthology.orchestra.run.smithi059.stdout: "log_data": "false", 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "bucket_index_max_shards": 11, 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "read_only": "false", 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "tier_type": "", 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "sync_from_all": "true", 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "sync_from": [], 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "redirect_zone": "" 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "placement_targets": [ 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: { 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "name": "default-placement", 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "tags": [], 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "storage_classes": [ 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: "STANDARD" 2024-04-26T15:57:41.260 INFO:teuthology.orchestra.run.smithi059.stdout: ] 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: "default_placement": "default-placement", 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: "realm_id": "5f31e43f-ac1c-48e4-a45d-d4f4ca882b4a", 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: "sync_policy": { 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: "groups": [] 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: "short_zone_ids": [ 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: { 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: "key": "c77933b7-fe75-4887-98df-c99f0c3dc1aa", 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: "val": 2727384348 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T15:57:41.261 INFO:teuthology.orchestra.run.smithi059.stdout: ] 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "master_zonegroup": "94a0e3d3-e849-4a13-a6e1-1c1b4ba527e7", 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "master_zone": "c77933b7-fe75-4887-98df-c99f0c3dc1aa", 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "period_config": { 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "bucket_quota": { 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "enabled": false, 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "check_on_raw": false, 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "max_size": -1, 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "max_size_kb": 0, 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "max_objects": -1 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "user_quota": { 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "enabled": false, 2024-04-26T15:57:41.262 INFO:teuthology.orchestra.run.smithi059.stdout: "check_on_raw": false, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "max_size": -1, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "max_size_kb": 0, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "max_objects": -1 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "user_ratelimit": { 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "max_read_ops": 0, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "max_write_ops": 0, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "max_read_bytes": 0, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "max_write_bytes": 0, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "enabled": false 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "bucket_ratelimit": { 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "max_read_ops": 0, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "max_write_ops": 0, 2024-04-26T15:57:41.263 INFO:teuthology.orchestra.run.smithi059.stdout: "max_read_bytes": 0, 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "max_write_bytes": 0, 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "enabled": false 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "anonymous_ratelimit": { 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "max_read_ops": 0, 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "max_write_ops": 0, 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "max_read_bytes": 0, 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "max_write_bytes": 0, 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "enabled": false 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "realm_id": "5f31e43f-ac1c-48e4-a45d-d4f4ca882b4a", 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "realm_name": "r", 2024-04-26T15:57:41.264 INFO:teuthology.orchestra.run.smithi059.stdout: "realm_epoch": 2 2024-04-26T15:57:41.265 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T15:57:41.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:41 smithi064 ceph-mon[22742]: pgmap v58: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-26T15:57:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:41 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-26T15:57:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:41 smithi064 ceph-mon[22742]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T15:57:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:41 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-26T15:57:41.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:41 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:41.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:41 smithi059 ceph-mon[26536]: pgmap v58: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-26T15:57:41.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:41 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-26T15:57:41.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:41 smithi059 ceph-mon[26536]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T15:57:41.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:41 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-26T15:57:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:41 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:41 smithi059 ceph-mon[21819]: pgmap v58: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-26T15:57:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:41 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-26T15:57:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:41 smithi059 ceph-mon[21819]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T15:57:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:41 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-26T15:57:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:41 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:41.994 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-04-26T15:57:42.319 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:42 smithi059 ceph-mon[26536]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T15:57:42.319 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:42 smithi059 ceph-mon[26536]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T15:57:42.320 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:42 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-26T15:57:42.320 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:42 smithi059 ceph-mon[26536]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T15:57:42.320 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:42 smithi059 ceph-mon[21819]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T15:57:42.320 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:42 smithi059 ceph-mon[21819]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T15:57:42.320 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:42 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-26T15:57:42.320 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:42 smithi059 ceph-mon[21819]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T15:57:42.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:42 smithi064 ceph-mon[22742]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T15:57:42.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:42 smithi064 ceph-mon[22742]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T15:57:42.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:42 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/718450404' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-26T15:57:42.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:42 smithi064 ceph-mon[22742]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T15:57:42.582 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T15:57:42.336Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000282274s 2024-04-26T15:57:43.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:43 smithi059 ceph-mon[21819]: pgmap v61: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-26T15:57:43.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:43 smithi059 ceph-mon[26536]: pgmap v61: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-26T15:57:43.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-mon[22742]: pgmap v61: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-26T15:57:43.735 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 systemd[1]: Stopping Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:57:43.735 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 bash[39501]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.735Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[37843]: ts=2024-04-26T15:57:43.736Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-26T15:57:44.021 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:43 smithi064 podman[39515]: 2024-04-26 15:57:43.746172023 +0000 UTC m=+0.116494998 container died 91f5db4c48b21e6f0d15f2237893369dd8398bb6387ee28571db99b5e39d8629 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:44.106 INFO:teuthology.orchestra.run.smithi059.stdout:Scheduled rgw.foo update... 2024-04-26T15:57:44.272 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 podman[39515]: 2024-04-26 15:57:44.227368438 +0000 UTC m=+0.597691413 container cleanup 91f5db4c48b21e6f0d15f2237893369dd8398bb6387ee28571db99b5e39d8629 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:44.272 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 bash[39515]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T15:57:44.272 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 bash[39544]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-26T15:57:44.694 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 podman[39529]: 2024-04-26 15:57:44.445277916 +0000 UTC m=+0.700309667 container remove 91f5db4c48b21e6f0d15f2237893369dd8398bb6387ee28571db99b5e39d8629 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:44.694 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T15:57:44.695 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 systemd[1]: Stopped Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:57:44.695 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 systemd[1]: Starting Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T15:57:44.695 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 podman[39613]: 2024-04-26 15:57:44.586743773 +0000 UTC m=+0.019942007 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-26T15:57:44.695 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 podman[39613]: 2024-04-26 15:57:44.695833607 +0000 UTC m=+0.129031829 container create 316a9bec1ec56d52a640251a534931570f96c5f6addc08a118f6d7090469dc73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:44.709 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph osd pool create foo' 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 podman[39613]: 2024-04-26 15:57:44.909381505 +0000 UTC m=+0.342579730 container init 316a9bec1ec56d52a640251a534931570f96c5f6addc08a118f6d7090469dc73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 podman[39613]: 2024-04-26 15:57:44.911966881 +0000 UTC m=+0.345165112 container start 316a9bec1ec56d52a640251a534931570f96c5f6addc08a118f6d7090469dc73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.937Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.937Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.937Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.937Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi064 (none))" 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.937Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.937Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.939Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.939Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.941Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-26T15:57:45.000 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.943Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T15:57:45.001 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.943Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.532µs 2024-04-26T15:57:45.001 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:44 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:44.943Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T15:57:45.001 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:45 smithi064 bash[39613]: 316a9bec1ec56d52a640251a534931570f96c5f6addc08a118f6d7090469dc73 2024-04-26T15:57:45.281 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: pgmap v62: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-04-26T15:57:45.281 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='client.14691 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:57:45.281 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: Saving service rgw.foo spec with placement count:2 2024-04-26T15:57:45.281 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.281 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.281 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:57:45.282 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.59:9093"}]: dispatch 2024-04-26T15:57:45.282 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.282 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:57:45.282 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.64:9095"}]: dispatch 2024-04-26T15:57:45.282 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.282 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:57:45.282 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.64:3000"}]: dispatch 2024-04-26T15:57:45.282 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.282 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:57:45.282 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:45.283 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:57:45.283 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: pgmap v62: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-04-26T15:57:45.283 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='client.14691 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:57:45.283 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: Saving service rgw.foo spec with placement count:2 2024-04-26T15:57:45.283 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.283 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.283 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:57:45.283 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.59:9093"}]: dispatch 2024-04-26T15:57:45.283 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.283 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:57:45.284 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.64:9095"}]: dispatch 2024-04-26T15:57:45.284 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.284 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:57:45.284 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.64:3000"}]: dispatch 2024-04-26T15:57:45.284 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.284 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:57:45.284 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:45.284 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:45 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:57:45.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:45 smithi064 systemd[1]: Started Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T15:57:45.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:45.165Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-26T15:57:45.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:45.165Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-26T15:57:45.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:45.165Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.891µs wal_replay_duration=222.782972ms total_replay_duration=222.827574ms 2024-04-26T15:57:45.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:45.168Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T15:57:45.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:45.168Z caller=main.go:947 level=info msg="TSDB started" 2024-04-26T15:57:45.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:45.168Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T15:57:45.438 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:45.192Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.731903ms db_storage=772ns remote_storage=1.659µs web_handler=470ns query_engine=869ns scrape=597.914µs scrape_sd=34.573µs notify=30.759µs notify_sd=12.871µs rules=22.716829ms 2024-04-26T15:57:45.438 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T15:57:45.192Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-26T15:57:45.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: pgmap v62: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-04-26T15:57:45.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='client.14691 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:57:45.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: Saving service rgw.foo spec with placement count:2 2024-04-26T15:57:45.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.59:9093"}]: dispatch 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.64:9095"}]: dispatch 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.64:3000"}]: dispatch 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:45.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:45 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:57:46.222 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:57:46.222 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.59:9093"}]: dispatch 2024-04-26T15:57:46.222 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:57:46.222 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.64:9095"}]: dispatch 2024-04-26T15:57:46.223 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:57:46.223 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.64:3000"}]: dispatch 2024-04-26T15:57:46.223 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:57:46.223 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.59:9093"}]: dispatch 2024-04-26T15:57:46.223 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:57:46.223 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.64:9095"}]: dispatch 2024-04-26T15:57:46.223 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:57:46.223 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:46 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.64:3000"}]: dispatch 2024-04-26T15:57:46.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:46 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T15:57:46.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:46 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.59:9093"}]: dispatch 2024-04-26T15:57:46.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:46 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T15:57:46.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:46 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.64:9095"}]: dispatch 2024-04-26T15:57:46.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:46 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T15:57:46.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:46 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.64:3000"}]: dispatch 2024-04-26T15:57:47.125 INFO:teuthology.orchestra.run.smithi059.stderr:pool 'foo' created 2024-04-26T15:57:47.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:47 smithi064 ceph-mon[22742]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 2 op/s 2024-04-26T15:57:47.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:47 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3128040578' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-26T15:57:47.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:47 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-26T15:57:47.478 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:47 smithi059 ceph-mon[26536]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 2 op/s 2024-04-26T15:57:47.479 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:47 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3128040578' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-26T15:57:47.479 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:47 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-26T15:57:47.479 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:47 smithi059 ceph-mon[21819]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 2 op/s 2024-04-26T15:57:47.479 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:47 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3128040578' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-26T15:57:47.479 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:47 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-26T15:57:47.899 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'rbd pool init foo' 2024-04-26T15:57:48.135 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:48 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-26T15:57:48.135 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:48 smithi059 ceph-mon[21819]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T15:57:48.407 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:48 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-26T15:57:48.407 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:48 smithi059 ceph-mon[26536]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T15:57:48.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:48 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-26T15:57:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:48 smithi064 ceph-mon[22742]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T15:57:49.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:49 smithi064 ceph-mon[22742]: pgmap v65: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 906 B/s wr, 2 op/s 2024-04-26T15:57:49.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:49 smithi064 ceph-mon[22742]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T15:57:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:49 smithi064 ceph-mon[22742]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T15:57:49.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:49 smithi059 ceph-mon[21819]: pgmap v65: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 906 B/s wr, 2 op/s 2024-04-26T15:57:49.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:49 smithi059 ceph-mon[21819]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T15:57:49.525 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:49 smithi059 ceph-mon[21819]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T15:57:49.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:49 smithi059 ceph-mon[26536]: pgmap v65: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 906 B/s wr, 2 op/s 2024-04-26T15:57:49.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:49 smithi059 ceph-mon[26536]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T15:57:49.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:49 smithi059 ceph-mon[26536]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T15:57:50.419 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 15:57:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T15:57:50.338Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.00181348s 2024-04-26T15:57:50.419 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:50 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2932565033' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-26T15:57:50.419 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:50 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-26T15:57:50.420 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:50 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2932565033' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-26T15:57:50.420 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:50 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-26T15:57:50.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:50 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2932565033' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-26T15:57:50.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:50 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-26T15:57:51.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:51 smithi064 ceph-mon[22742]: pgmap v68: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:51 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-26T15:57:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:51 smithi064 ceph-mon[22742]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T15:57:51.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:51 smithi059 ceph-mon[26536]: pgmap v68: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:51.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:51 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-26T15:57:51.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:51 smithi059 ceph-mon[26536]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T15:57:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:51 smithi059 ceph-mon[21819]: pgmap v68: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:51 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-26T15:57:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:51 smithi059 ceph-mon[21819]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T15:57:52.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:52 smithi064 ceph-mon[22742]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T15:57:52.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:52 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:52.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:52 smithi059 ceph-mon[21819]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T15:57:52.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:52 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:52.518 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:52 smithi059 ceph-mon[26536]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T15:57:52.518 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:52 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:52.873 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch apply iscsi foo u p' 2024-04-26T15:57:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T15:57:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-26T15:57:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:53 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:53.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:53.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T15:57:53.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.449 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.449 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:53.449 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:53.449 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-26T15:57:53.449 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:53.449 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:53 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:54.328 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:54 smithi059 ceph-mon[21819]: Saving service rgw.foo spec with placement count:2 2024-04-26T15:57:54.328 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:54 smithi059 ceph-mon[21819]: Deploying daemon rgw.foo.smithi059.dvtzbj on smithi059 2024-04-26T15:57:54.328 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:54 smithi059 ceph-mon[26536]: Saving service rgw.foo spec with placement count:2 2024-04-26T15:57:54.328 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:54 smithi059 ceph-mon[26536]: Deploying daemon rgw.foo.smithi059.dvtzbj on smithi059 2024-04-26T15:57:54.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:54 smithi064 ceph-mon[22742]: Saving service rgw.foo spec with placement count:2 2024-04-26T15:57:54.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:54 smithi064 ceph-mon[22742]: Deploying daemon rgw.foo.smithi059.dvtzbj on smithi059 2024-04-26T15:57:54.763 INFO:teuthology.orchestra.run.smithi059.stdout:Scheduled iscsi.foo update... 2024-04-26T15:57:55.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:57:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:57:54] "GET /metrics HTTP/1.1" 200 197860 "" "Prometheus/2.33.4" 2024-04-26T15:57:55.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:55 smithi059 ceph-mon[21819]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 412 B/s wr, 0 op/s 2024-04-26T15:57:55.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:55 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:55.433 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:55 smithi059 ceph-mon[26536]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 412 B/s wr, 0 op/s 2024-04-26T15:57:55.433 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:55 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:55.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:55 smithi064 ceph-mon[22742]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 412 B/s wr, 0 op/s 2024-04-26T15:57:55.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:55 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:55.640 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'sleep 180' 2024-04-26T15:57:56.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:56 smithi064 ceph-mon[22742]: from='client.24616 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:57:56.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:56 smithi064 ceph-mon[22742]: Saving service iscsi.foo spec with placement count:1 2024-04-26T15:57:56.498 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:56 smithi059 ceph-mon[21819]: from='client.24616 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:57:56.498 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:56 smithi059 ceph-mon[21819]: Saving service iscsi.foo spec with placement count:1 2024-04-26T15:57:56.498 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:56 smithi059 ceph-mon[26536]: from='client.24616 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T15:57:56.498 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:56 smithi059 ceph-mon[26536]: Saving service iscsi.foo spec with placement count:1 2024-04-26T15:57:57.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:57 smithi064 ceph-mon[22742]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-04-26T15:57:57.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:57 smithi064 ceph-mon[22742]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T15:57:57.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:57 smithi059 ceph-mon[21819]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-04-26T15:57:57.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:57 smithi059 ceph-mon[21819]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T15:57:57.439 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:57 smithi059 ceph-mon[26536]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-04-26T15:57:57.439 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:57 smithi059 ceph-mon[26536]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T15:57:59.173 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:57:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:57:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:57:59.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:59 smithi064 ceph-mon[22742]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 294 B/s wr, 0 op/s 2024-04-26T15:57:59.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:59 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:59.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:59 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:59.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:59 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:59.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:59 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-26T15:57:59.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:59 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:59.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:57:59 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:59.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[21819]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 294 B/s wr, 0 op/s 2024-04-26T15:57:59.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:59.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:59.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:59.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-26T15:57:59.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:59.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:57:59.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[26536]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 294 B/s wr, 0 op/s 2024-04-26T15:57:59.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:59.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:59.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T15:57:59.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-26T15:57:59.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:57:59.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:57:59 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:00.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:00 smithi064 ceph-mon[22742]: Deploying daemon rgw.foo.smithi064.sqspir on smithi064 2024-04-26T15:58:00.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:00 smithi059 ceph-mon[26536]: Deploying daemon rgw.foo.smithi064.sqspir on smithi064 2024-04-26T15:58:00.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:00 smithi059 ceph-mon[21819]: Deploying daemon rgw.foo.smithi064.sqspir on smithi064 2024-04-26T15:58:01.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:01 smithi059 ceph-mon[21819]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 4.4 KiB/s wr, 137 op/s 2024-04-26T15:58:01.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:01 smithi059 ceph-mon[26536]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 4.4 KiB/s wr, 137 op/s 2024-04-26T15:58:01.362 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:01 smithi064 ceph-mon[22742]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 4.4 KiB/s wr, 137 op/s 2024-04-26T15:58:02.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:02 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:02.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:02 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:58:02.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:02 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:02.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:02 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:58:02.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:02 smithi064 ceph-mon[22742]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 111 op/s 2024-04-26T15:58:02.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:02 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:02 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:58:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:02 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:02 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:58:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:02 smithi059 ceph-mon[21819]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 111 op/s 2024-04-26T15:58:02.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:02 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:02.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:02 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:58:02.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:02 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:02.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:02 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:58:02.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:02 smithi059 ceph-mon[26536]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 111 op/s 2024-04-26T15:58:04.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:04 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:04.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:04 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:04.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:04 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:58:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:04] "GET /metrics HTTP/1.1" 200 197860 "" "Prometheus/2.33.4" 2024-04-26T15:58:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[21819]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-04-26T15:58:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T15:58:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T15:58:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-26T15:58:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:05.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[26536]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-04-26T15:58:05.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T15:58:05.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T15:58:05.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-26T15:58:05.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:05 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:05.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:05 smithi064 ceph-mon[22742]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-04-26T15:58:05.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:05 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:05 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:05 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:05 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:05 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:05.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:05 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T15:58:05.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:05 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T15:58:05.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:05 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-26T15:58:05.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:05 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:06.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:06 smithi064 ceph-mon[22742]: Checking dashboard <-> RGW credentials 2024-04-26T15:58:06.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:06 smithi064 ceph-mon[22742]: Checking pool "foo" exists for service iscsi.foo 2024-04-26T15:58:06.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:06 smithi064 ceph-mon[22742]: Deploying daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T15:58:06.502 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:06 smithi059 ceph-mon[21819]: Checking dashboard <-> RGW credentials 2024-04-26T15:58:06.502 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:06 smithi059 ceph-mon[21819]: Checking pool "foo" exists for service iscsi.foo 2024-04-26T15:58:06.502 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:06 smithi059 ceph-mon[21819]: Deploying daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T15:58:06.503 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:06 smithi059 ceph-mon[26536]: Checking dashboard <-> RGW credentials 2024-04-26T15:58:06.503 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:06 smithi059 ceph-mon[26536]: Checking pool "foo" exists for service iscsi.foo 2024-04-26T15:58:06.503 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:06 smithi059 ceph-mon[26536]: Deploying daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T15:58:07.430 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:07 smithi059 ceph-mon[26536]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-04-26T15:58:07.430 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:07 smithi059 ceph-mon[26536]: mgrmap e20: y(active, since 118s), standbys: x 2024-04-26T15:58:07.430 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:07 smithi059 ceph-mon[21819]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-04-26T15:58:07.430 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:07 smithi059 ceph-mon[21819]: mgrmap e20: y(active, since 118s), standbys: x 2024-04-26T15:58:07.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:07 smithi064 ceph-mon[22742]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-04-26T15:58:07.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:07 smithi064 ceph-mon[22742]: mgrmap e20: y(active, since 118s), standbys: x 2024-04-26T15:58:08.250 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-26T15:58:08.250 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.250 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-26T15:58:08.250 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.250 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.250 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.250 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.251 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:58:08.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:08.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:58:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:58:08.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:58:08.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:08.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:58:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:58:09.184 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[21819]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 101 KiB/s rd, 3.1 KiB/s wr, 193 op/s 2024-04-26T15:58:09.184 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-04-26T15:58:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:09 smithi064 ceph-mon[22742]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 101 KiB/s rd, 3.1 KiB/s wr, 193 op/s 2024-04-26T15:58:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-04-26T15:58:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-26T15:58:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-26T15:58:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-26T15:58:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:09 smithi064 ceph-mon[22742]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T15:58:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:09 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1196140899' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T15:58:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:09 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3390991898' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/434932617"}]: dispatch 2024-04-26T15:58:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[21819]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1196140899' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3390991898' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/434932617"}]: dispatch 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[26536]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 101 KiB/s rd, 3.1 KiB/s wr, 193 op/s 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-26T15:58:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[26536]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T15:58:09.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1196140899' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T15:58:09.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:09 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3390991898' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/434932617"}]: dispatch 2024-04-26T15:58:10.501 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:10 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3390991898' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/434932617"}]': finished 2024-04-26T15:58:10.501 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:10 smithi064 ceph-mon[22742]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T15:58:10.501 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:10 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/956988344' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2069431914"}]: dispatch 2024-04-26T15:58:10.501 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:10 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2069431914"}]: dispatch 2024-04-26T15:58:10.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:10 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3390991898' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/434932617"}]': finished 2024-04-26T15:58:10.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:10 smithi059 ceph-mon[21819]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T15:58:10.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:10 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/956988344' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2069431914"}]: dispatch 2024-04-26T15:58:10.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:10 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2069431914"}]: dispatch 2024-04-26T15:58:10.548 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:10 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3390991898' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/434932617"}]': finished 2024-04-26T15:58:10.548 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:10 smithi059 ceph-mon[26536]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T15:58:10.548 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:10 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/956988344' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2069431914"}]: dispatch 2024-04-26T15:58:10.548 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:10 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2069431914"}]: dispatch 2024-04-26T15:58:11.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:11 smithi064 ceph-mon[22742]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 511 B/s wr, 134 op/s 2024-04-26T15:58:11.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:11 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2069431914"}]': finished 2024-04-26T15:58:11.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:11 smithi064 ceph-mon[22742]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T15:58:11.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:11 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/214570647' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3047449642"}]: dispatch 2024-04-26T15:58:11.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:11 smithi059 ceph-mon[21819]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 511 B/s wr, 134 op/s 2024-04-26T15:58:11.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:11 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2069431914"}]': finished 2024-04-26T15:58:11.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:11 smithi059 ceph-mon[21819]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T15:58:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:11 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/214570647' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3047449642"}]: dispatch 2024-04-26T15:58:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:11 smithi059 ceph-mon[26536]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 511 B/s wr, 134 op/s 2024-04-26T15:58:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:11 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2069431914"}]': finished 2024-04-26T15:58:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:11 smithi059 ceph-mon[26536]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T15:58:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:11 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/214570647' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3047449642"}]: dispatch 2024-04-26T15:58:12.521 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/214570647' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3047449642"}]': finished 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/724547485' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/635818739"}]: dispatch 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi059"}]: dispatch 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:12.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/214570647' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3047449642"}]': finished 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/724547485' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/635818739"}]: dispatch 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi059"}]: dispatch 2024-04-26T15:58:12.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:58:12.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:12.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:58:12.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:12 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/214570647' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3047449642"}]': finished 2024-04-26T15:58:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T15:58:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/724547485' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/635818739"}]: dispatch 2024-04-26T15:58:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T15:58:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T15:58:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi059"}]: dispatch 2024-04-26T15:58:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:58:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:58:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:58:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:12 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:58:13.527 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T15:58:13.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[21819]: Adding iSCSI gateway http://:@172.21.15.59:5000 to Dashboard 2024-04-26T15:58:13.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T15:58:13.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi059"}]: dispatch 2024-04-26T15:58:13.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[21819]: Checking dashboard <-> RGW credentials 2024-04-26T15:58:13.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[21819]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s 2024-04-26T15:58:13.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/724547485' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/635818739"}]': finished 2024-04-26T15:58:13.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[21819]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T15:58:13.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3985449124' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3932195089"}]: dispatch 2024-04-26T15:58:13.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3932195089"}]: dispatch 2024-04-26T15:58:13.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T15:58:13.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[26536]: Adding iSCSI gateway http://:@172.21.15.59:5000 to Dashboard 2024-04-26T15:58:13.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T15:58:13.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi059"}]: dispatch 2024-04-26T15:58:13.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[26536]: Checking dashboard <-> RGW credentials 2024-04-26T15:58:13.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[26536]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s 2024-04-26T15:58:13.531 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/724547485' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/635818739"}]': finished 2024-04-26T15:58:13.531 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[26536]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T15:58:13.531 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3985449124' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3932195089"}]: dispatch 2024-04-26T15:58:13.531 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:13 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3932195089"}]: dispatch 2024-04-26T15:58:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:13 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T15:58:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:13 smithi064 ceph-mon[22742]: Adding iSCSI gateway http://:@172.21.15.59:5000 to Dashboard 2024-04-26T15:58:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:13 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T15:58:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:13 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi059"}]: dispatch 2024-04-26T15:58:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:13 smithi064 ceph-mon[22742]: Checking dashboard <-> RGW credentials 2024-04-26T15:58:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:13 smithi064 ceph-mon[22742]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s 2024-04-26T15:58:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:13 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/724547485' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/635818739"}]': finished 2024-04-26T15:58:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:13 smithi064 ceph-mon[22742]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T15:58:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:13 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3985449124' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3932195089"}]: dispatch 2024-04-26T15:58:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:13 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3932195089"}]: dispatch 2024-04-26T15:58:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3932195089"}]': finished 2024-04-26T15:58:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-mon[21819]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T15:58:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1599738210' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1212694920"}]: dispatch 2024-04-26T15:58:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1212694920"}]: dispatch 2024-04-26T15:58:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-mon[21819]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-26T15:58:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3932195089"}]': finished 2024-04-26T15:58:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-mon[26536]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T15:58:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1599738210' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1212694920"}]: dispatch 2024-04-26T15:58:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1212694920"}]: dispatch 2024-04-26T15:58:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-mon[26536]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-26T15:58:14.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:14 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3932195089"}]': finished 2024-04-26T15:58:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:14 smithi064 ceph-mon[22742]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T15:58:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:14 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1599738210' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1212694920"}]: dispatch 2024-04-26T15:58:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:14 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1212694920"}]: dispatch 2024-04-26T15:58:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:14 smithi064 ceph-mon[22742]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-26T15:58:15.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:58:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:14] "GET /metrics HTTP/1.1" 200 207938 "" "Prometheus/2.33.4" 2024-04-26T15:58:15.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:15 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1212694920"}]': finished 2024-04-26T15:58:15.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:15 smithi059 ceph-mon[26536]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T15:58:15.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:15 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1236182307' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1573688154"}]: dispatch 2024-04-26T15:58:15.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:15 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1573688154"}]: dispatch 2024-04-26T15:58:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:15 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1212694920"}]': finished 2024-04-26T15:58:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:15 smithi059 ceph-mon[21819]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T15:58:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:15 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1236182307' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1573688154"}]: dispatch 2024-04-26T15:58:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:15 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1573688154"}]: dispatch 2024-04-26T15:58:15.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:15 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1212694920"}]': finished 2024-04-26T15:58:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:15 smithi064 ceph-mon[22742]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T15:58:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:15 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1236182307' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1573688154"}]: dispatch 2024-04-26T15:58:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:15 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1573688154"}]: dispatch 2024-04-26T15:58:16.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:16 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1573688154"}]': finished 2024-04-26T15:58:16.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:16 smithi059 ceph-mon[26536]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T15:58:16.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:16 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1923885284' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2655313575"}]: dispatch 2024-04-26T15:58:16.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:16 smithi059 ceph-mon[26536]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-26T15:58:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:16 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1573688154"}]': finished 2024-04-26T15:58:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:16 smithi059 ceph-mon[21819]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T15:58:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:16 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1923885284' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2655313575"}]: dispatch 2024-04-26T15:58:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:16 smithi059 ceph-mon[21819]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-26T15:58:16.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:16 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1573688154"}]': finished 2024-04-26T15:58:16.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:16 smithi064 ceph-mon[22742]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T15:58:16.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:16 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1923885284' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2655313575"}]: dispatch 2024-04-26T15:58:16.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:16 smithi064 ceph-mon[22742]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-26T15:58:17.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:17 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1923885284' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2655313575"}]': finished 2024-04-26T15:58:17.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:17 smithi059 ceph-mon[26536]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T15:58:17.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:17 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1613155243' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1094121986"}]: dispatch 2024-04-26T15:58:17.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:17 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1923885284' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2655313575"}]': finished 2024-04-26T15:58:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:17 smithi059 ceph-mon[21819]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T15:58:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:17 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1613155243' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1094121986"}]: dispatch 2024-04-26T15:58:17.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:17 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1923885284' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2655313575"}]': finished 2024-04-26T15:58:17.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:17 smithi064 ceph-mon[22742]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T15:58:17.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:17 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1613155243' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1094121986"}]: dispatch 2024-04-26T15:58:18.831 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:18 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1613155243' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1094121986"}]': finished 2024-04-26T15:58:18.831 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:18 smithi064 ceph-mon[22742]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T15:58:18.831 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:18 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3856089310' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1707008100"}]: dispatch 2024-04-26T15:58:18.832 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:18 smithi064 ceph-mon[22742]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:58:18.832 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:18 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:18.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:18 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1613155243' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1094121986"}]': finished 2024-04-26T15:58:18.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:18 smithi059 ceph-mon[21819]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T15:58:18.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:18 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3856089310' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1707008100"}]: dispatch 2024-04-26T15:58:18.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:18 smithi059 ceph-mon[21819]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:58:18.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:18 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:18 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1613155243' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1094121986"}]': finished 2024-04-26T15:58:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:18 smithi059 ceph-mon[26536]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T15:58:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:18 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3856089310' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1707008100"}]: dispatch 2024-04-26T15:58:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:18 smithi059 ceph-mon[26536]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-04-26T15:58:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:18 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:19.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:58:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:58:19.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:19 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3856089310' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1707008100"}]': finished 2024-04-26T15:58:19.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:19 smithi059 ceph-mon[21819]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T15:58:19.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:19 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3466574935' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/890201972"}]: dispatch 2024-04-26T15:58:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:19 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3856089310' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1707008100"}]': finished 2024-04-26T15:58:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:19 smithi059 ceph-mon[26536]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T15:58:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:19 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3466574935' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/890201972"}]: dispatch 2024-04-26T15:58:19.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:19 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3856089310' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1707008100"}]': finished 2024-04-26T15:58:19.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:19 smithi064 ceph-mon[22742]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T15:58:19.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:19 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3466574935' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/890201972"}]: dispatch 2024-04-26T15:58:20.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:20 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3466574935' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/890201972"}]': finished 2024-04-26T15:58:20.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:20 smithi059 ceph-mon[21819]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T15:58:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:20 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3837655743' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1212694920"}]: dispatch 2024-04-26T15:58:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:20 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1212694920"}]: dispatch 2024-04-26T15:58:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:20 smithi059 ceph-mon[21819]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:20 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3466574935' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/890201972"}]': finished 2024-04-26T15:58:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:20 smithi059 ceph-mon[26536]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T15:58:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:20 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3837655743' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1212694920"}]: dispatch 2024-04-26T15:58:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:20 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1212694920"}]: dispatch 2024-04-26T15:58:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:20 smithi059 ceph-mon[26536]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:20.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:20 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3466574935' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/890201972"}]': finished 2024-04-26T15:58:20.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:20 smithi064 ceph-mon[22742]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T15:58:20.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:20 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3837655743' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1212694920"}]: dispatch 2024-04-26T15:58:20.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:20 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1212694920"}]: dispatch 2024-04-26T15:58:20.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:20 smithi064 ceph-mon[22742]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:21.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:21 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1212694920"}]': finished 2024-04-26T15:58:21.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:21 smithi059 ceph-mon[26536]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T15:58:21.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:21 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3142227813' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/890201972"}]: dispatch 2024-04-26T15:58:21.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:21 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/890201972"}]: dispatch 2024-04-26T15:58:21.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:21 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1212694920"}]': finished 2024-04-26T15:58:21.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:21 smithi059 ceph-mon[21819]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T15:58:21.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:21 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3142227813' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/890201972"}]: dispatch 2024-04-26T15:58:21.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:21 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/890201972"}]: dispatch 2024-04-26T15:58:21.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:21 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1212694920"}]': finished 2024-04-26T15:58:21.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:21 smithi064 ceph-mon[22742]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T15:58:21.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:21 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3142227813' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/890201972"}]: dispatch 2024-04-26T15:58:21.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:21 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/890201972"}]: dispatch 2024-04-26T15:58:22.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:22 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/890201972"}]': finished 2024-04-26T15:58:22.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:22 smithi059 ceph-mon[21819]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T15:58:22.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:22 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2116595173' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/434932617"}]: dispatch 2024-04-26T15:58:22.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:22 smithi059 ceph-mon[21819]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:22.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:22 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/890201972"}]': finished 2024-04-26T15:58:22.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:22 smithi059 ceph-mon[26536]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T15:58:22.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:22 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2116595173' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/434932617"}]: dispatch 2024-04-26T15:58:22.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:22 smithi059 ceph-mon[26536]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:22.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:22 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/890201972"}]': finished 2024-04-26T15:58:22.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:22 smithi064 ceph-mon[22742]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T15:58:22.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:22 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2116595173' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/434932617"}]: dispatch 2024-04-26T15:58:22.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:22 smithi064 ceph-mon[22742]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:23.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:23 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2116595173' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/434932617"}]': finished 2024-04-26T15:58:23.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:23 smithi059 ceph-mon[21819]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T15:58:23.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:23 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3574800265' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3060676991"}]: dispatch 2024-04-26T15:58:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:23 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2116595173' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/434932617"}]': finished 2024-04-26T15:58:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:23 smithi059 ceph-mon[26536]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T15:58:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:23 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3574800265' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3060676991"}]: dispatch 2024-04-26T15:58:23.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:23 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2116595173' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/434932617"}]': finished 2024-04-26T15:58:23.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:23 smithi064 ceph-mon[22742]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T15:58:23.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:23 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3574800265' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3060676991"}]: dispatch 2024-04-26T15:58:24.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:24 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3574800265' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3060676991"}]': finished 2024-04-26T15:58:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:24 smithi064 ceph-mon[22742]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T15:58:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:24 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1918807593' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2924841679"}]: dispatch 2024-04-26T15:58:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:24 smithi064 ceph-mon[22742]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:24.981 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:24 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3574800265' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3060676991"}]': finished 2024-04-26T15:58:24.981 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:24 smithi059 ceph-mon[21819]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T15:58:24.982 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:24 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1918807593' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2924841679"}]: dispatch 2024-04-26T15:58:24.982 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:24 smithi059 ceph-mon[21819]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:24.982 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:24 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3574800265' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3060676991"}]': finished 2024-04-26T15:58:24.982 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:24 smithi059 ceph-mon[26536]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T15:58:24.982 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:24 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1918807593' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2924841679"}]: dispatch 2024-04-26T15:58:24.982 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:24 smithi059 ceph-mon[26536]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:25.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:58:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:24] "GET /metrics HTTP/1.1" 200 208046 "" "Prometheus/2.33.4" 2024-04-26T15:58:25.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:25 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1918807593' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2924841679"}]': finished 2024-04-26T15:58:25.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:25 smithi064 ceph-mon[22742]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T15:58:26.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:25 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1918807593' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2924841679"}]': finished 2024-04-26T15:58:26.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:25 smithi059 ceph-mon[21819]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T15:58:26.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:25 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1918807593' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2924841679"}]': finished 2024-04-26T15:58:26.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:25 smithi059 ceph-mon[26536]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T15:58:26.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:26 smithi064 ceph-mon[22742]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T15:58:27.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:26 smithi059 ceph-mon[21819]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T15:58:27.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:26 smithi059 ceph-mon[26536]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T15:58:29.159 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:58:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:58:29.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:29 smithi064 ceph-mon[22742]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:29.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:29 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:29.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:29 smithi059 ceph-mon[21819]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:29 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:29 smithi059 ceph-mon[26536]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:29 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:31.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:31 smithi064 ceph-mon[22742]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T15:58:31.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:31 smithi059 ceph-mon[26536]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T15:58:31.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:31 smithi059 ceph-mon[21819]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T15:58:33.432 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:33 smithi064 ceph-mon[22742]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:33 smithi059 ceph-mon[26536]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:33 smithi059 ceph-mon[21819]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:35.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:35 smithi059 ceph-mon[26536]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T15:58:35.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:35 smithi059 ceph-mon[21819]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T15:58:35.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:58:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:34] "GET /metrics HTTP/1.1" 200 208046 "" "Prometheus/2.33.4" 2024-04-26T15:58:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:35 smithi064 ceph-mon[22742]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T15:58:37.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:37 smithi064 ceph-mon[22742]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 0 op/s 2024-04-26T15:58:37.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:37 smithi059 ceph-mon[21819]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 0 op/s 2024-04-26T15:58:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:37 smithi059 ceph-mon[26536]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 0 op/s 2024-04-26T15:58:39.123 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:58:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:58:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:39 smithi064 ceph-mon[22742]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:39 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:39.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:39 smithi059 ceph-mon[21819]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:39.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:39 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:39.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:39 smithi059 ceph-mon[26536]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:39.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:39 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:41.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:41 smithi064 ceph-mon[22742]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:41.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:41 smithi059 ceph-mon[21819]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:41 smithi059 ceph-mon[26536]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:43.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:43 smithi064 ceph-mon[22742]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:43.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:43 smithi059 ceph-mon[21819]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:43.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:43 smithi059 ceph-mon[26536]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:45.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:45 smithi059 ceph-mon[21819]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:45.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:58:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:44] "GET /metrics HTTP/1.1" 200 207977 "" "Prometheus/2.33.4" 2024-04-26T15:58:45.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:45 smithi059 ceph-mon[26536]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:45.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:45 smithi064 ceph-mon[22742]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:47.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:47 smithi064 ceph-mon[22742]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:47.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:47 smithi059 ceph-mon[21819]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:47.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:47 smithi059 ceph-mon[26536]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:49.125 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:58:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:58:49.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:49 smithi064 ceph-mon[22742]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:49 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:49.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:49 smithi059 ceph-mon[21819]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:49.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:49 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:49.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:49 smithi059 ceph-mon[26536]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:49.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:49 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:58:51.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:51 smithi064 ceph-mon[22742]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:51.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:51 smithi059 ceph-mon[26536]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:51 smithi059 ceph-mon[21819]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:53.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:53 smithi064 ceph-mon[22742]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:53.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:53 smithi059 ceph-mon[26536]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:53.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:53 smithi059 ceph-mon[21819]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:55.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:55 smithi059 ceph-mon[26536]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:55.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:55 smithi059 ceph-mon[21819]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:55.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:58:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:54] "GET /metrics HTTP/1.1" 200 207954 "" "Prometheus/2.33.4" 2024-04-26T15:58:55.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:55 smithi064 ceph-mon[22742]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:58:57.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:57 smithi064 ceph-mon[22742]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:57.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:57 smithi059 ceph-mon[26536]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:57 smithi059 ceph-mon[21819]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:59.125 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:58:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:58:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:58:59.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:58:59 smithi064 ceph-mon[22742]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:59.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:58:59 smithi059 ceph-mon[21819]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:58:59.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:58:59 smithi059 ceph-mon[26536]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:00.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:00 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:00.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:00 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:00 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:01.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:01 smithi064 ceph-mon[22742]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:01.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:01 smithi059 ceph-mon[26536]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:01.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:01 smithi059 ceph-mon[21819]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:03.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:03 smithi064 ceph-mon[22742]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:03.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:03 smithi059 ceph-mon[26536]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:03.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:03 smithi059 ceph-mon[21819]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:05.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:05 smithi059 ceph-mon[26536]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:05.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:05 smithi059 ceph-mon[21819]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:05.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:59:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:04] "GET /metrics HTTP/1.1" 200 207954 "" "Prometheus/2.33.4" 2024-04-26T15:59:05.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:05 smithi064 ceph-mon[22742]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:07.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:07 smithi064 ceph-mon[22742]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:07.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:07 smithi059 ceph-mon[21819]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:07.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:07 smithi059 ceph-mon[26536]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:09.130 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:59:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:59:09.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:09 smithi064 ceph-mon[22742]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:09.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:59:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:59:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:09 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:59:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:09 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:59:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:09 smithi059 ceph-mon[21819]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:59:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:59:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:09 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:59:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:09 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:59:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:09 smithi059 ceph-mon[26536]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:59:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T15:59:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:09 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:59:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:09 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T15:59:10.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:10 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:10.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:10 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:10.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:10 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:11.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:11 smithi064 ceph-mon[22742]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:11.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:11 smithi059 ceph-mon[26536]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:11.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:11 smithi059 ceph-mon[21819]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:12.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:12 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:59:12.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:12 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:59:12.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:12 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:59:12.194 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:12 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:59:12.194 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:12 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:59:12.194 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:12 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:59:12.194 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:12 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T15:59:12.195 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:12 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T15:59:12.195 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:12 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T15:59:13.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:13 smithi059 ceph-mon[26536]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:13.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:13 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:59:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:13 smithi059 ceph-mon[21819]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:13 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:59:13.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:13 smithi064 ceph-mon[22742]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:13.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:13 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T15:59:15.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:15 smithi059 ceph-mon[26536]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:15.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:15 smithi059 ceph-mon[21819]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:15.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:59:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:14] "GET /metrics HTTP/1.1" 200 207929 "" "Prometheus/2.33.4" 2024-04-26T15:59:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:15 smithi064 ceph-mon[22742]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:17.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:17 smithi059 ceph-mon[26536]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:17.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:17 smithi059 ceph-mon[21819]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:17 smithi064 ceph-mon[22742]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:19.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:59:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:59:19.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:19 smithi059 ceph-mon[26536]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:19.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:19 smithi059 ceph-mon[21819]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:19.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:19 smithi064 ceph-mon[22742]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:20.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:20 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:20.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:20 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:20 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:21 smithi059 ceph-mon[21819]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:21.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:21 smithi059 ceph-mon[26536]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:21.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:21 smithi064 ceph-mon[22742]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:23.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:23 smithi064 ceph-mon[22742]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:23.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:23 smithi059 ceph-mon[21819]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:23.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:23 smithi059 ceph-mon[26536]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:25.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:25 smithi059 ceph-mon[26536]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:25.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:25 smithi059 ceph-mon[21819]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:25.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:59:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:24] "GET /metrics HTTP/1.1" 200 207928 "" "Prometheus/2.33.4" 2024-04-26T15:59:25.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:25 smithi064 ceph-mon[22742]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:27.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:27 smithi064 ceph-mon[22742]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:27.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:27 smithi059 ceph-mon[26536]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:27 smithi059 ceph-mon[21819]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:29.138 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:59:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:59:29.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:29 smithi064 ceph-mon[22742]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:29.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:29 smithi059 ceph-mon[21819]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:29.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:29 smithi059 ceph-mon[26536]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:30.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:30 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:30.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:30 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:30.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:30 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:31.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:31 smithi064 ceph-mon[22742]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:31.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:31 smithi059 ceph-mon[21819]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:31.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:31 smithi059 ceph-mon[26536]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:33 smithi064 ceph-mon[22742]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:33.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:33 smithi059 ceph-mon[21819]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:33.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:33 smithi059 ceph-mon[26536]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:35.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:59:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:34] "GET /metrics HTTP/1.1" 200 207928 "" "Prometheus/2.33.4" 2024-04-26T15:59:35.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:35 smithi059 ceph-mon[21819]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:35.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:35 smithi059 ceph-mon[26536]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:35 smithi064 ceph-mon[22742]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:37.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:37 smithi064 ceph-mon[22742]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:37.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:37 smithi059 ceph-mon[26536]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:37.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:37 smithi059 ceph-mon[21819]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:39.184 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:59:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:59:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:39 smithi064 ceph-mon[22742]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:39.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:39 smithi059 ceph-mon[26536]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:39.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:39 smithi059 ceph-mon[21819]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:40.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:40 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:40 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:40 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:41.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:41 smithi059 ceph-mon[21819]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:41 smithi059 ceph-mon[26536]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:41.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:41 smithi064 ceph-mon[22742]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:43.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:43 smithi064 ceph-mon[22742]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:43.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:43 smithi059 ceph-mon[21819]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:43.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:43 smithi059 ceph-mon[26536]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:45.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:45 smithi059 ceph-mon[21819]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:45.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:45 smithi059 ceph-mon[26536]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:45.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:59:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:44] "GET /metrics HTTP/1.1" 200 207911 "" "Prometheus/2.33.4" 2024-04-26T15:59:45.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:45 smithi064 ceph-mon[22742]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:47.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:47 smithi064 ceph-mon[22742]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:47.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:47 smithi059 ceph-mon[26536]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:47 smithi059 ceph-mon[21819]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:49.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:59:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:59:49.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:49 smithi059 ceph-mon[26536]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:49.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:49 smithi059 ceph-mon[21819]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:49.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:49 smithi064 ceph-mon[22742]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:50.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:50 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:50.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:50 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:50.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:50 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T15:59:51.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:51 smithi059 ceph-mon[26536]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:51 smithi059 ceph-mon[21819]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:51.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:51 smithi064 ceph-mon[22742]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:53 smithi064 ceph-mon[22742]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:53.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:53 smithi059 ceph-mon[26536]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:53 smithi059 ceph-mon[21819]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:55.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:55 smithi059 ceph-mon[26536]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:55.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:55 smithi059 ceph-mon[21819]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:55.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 15:59:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:54] "GET /metrics HTTP/1.1" 200 207921 "" "Prometheus/2.33.4" 2024-04-26T15:59:55.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:55 smithi064 ceph-mon[22742]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T15:59:57.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:57 smithi059 ceph-mon[21819]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:57.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:57 smithi059 ceph-mon[26536]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:57 smithi064 ceph-mon[22742]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:59.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 15:59:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:15:59:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T15:59:59.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 15:59:59 smithi059 ceph-mon[21819]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:59.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 15:59:59 smithi059 ceph-mon[26536]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T15:59:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 15:59:59 smithi064 ceph-mon[22742]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:00.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:00 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:00.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:00 smithi059 ceph-mon[26536]: overall HEALTH_OK 2024-04-26T16:00:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:00 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:00 smithi059 ceph-mon[21819]: overall HEALTH_OK 2024-04-26T16:00:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:00 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:00.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:00 smithi064 ceph-mon[22742]: overall HEALTH_OK 2024-04-26T16:00:01.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:01 smithi059 ceph-mon[21819]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:01 smithi059 ceph-mon[26536]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:01 smithi064 ceph-mon[22742]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:03.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:03 smithi064 ceph-mon[22742]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:03.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:03 smithi059 ceph-mon[21819]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:03 smithi059 ceph-mon[26536]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:05.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:05 smithi059 ceph-mon[21819]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:05.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:05 smithi059 ceph-mon[26536]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:05.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:00:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:04] "GET /metrics HTTP/1.1" 200 207921 "" "Prometheus/2.33.4" 2024-04-26T16:00:05.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:05 smithi064 ceph-mon[22742]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:07.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:07 smithi059 ceph-mon[21819]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:07.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:07 smithi059 ceph-mon[26536]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:07.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:07 smithi064 ceph-mon[22742]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:08.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:00:08.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:00:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:00:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:00:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:00:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:00:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:00:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:00:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:00:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:00:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:00:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:00:09.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:00:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:00:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:09 smithi059 ceph-mon[21819]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:09 smithi059 ceph-mon[26536]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:09 smithi064 ceph-mon[22742]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:10.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:10 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:10.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:10 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:10.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:10 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:11 smithi059 ceph-mon[26536]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:11.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:11 smithi059 ceph-mon[21819]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:11 smithi064 ceph-mon[22742]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:13 smithi064 ceph-mon[22742]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:13 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:00:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:13 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:00:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:13 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:00:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:13 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:00:13.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:13 smithi059 ceph-mon[26536]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:13.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:13 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:00:13.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:13 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:00:13.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:13 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:00:13.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:13 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:00:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:13 smithi059 ceph-mon[21819]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:13 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:00:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:13 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:00:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:13 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:00:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:13 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:00:15.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:15 smithi059 ceph-mon[26536]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:15.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:15 smithi059 ceph-mon[21819]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:15.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:00:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:14] "GET /metrics HTTP/1.1" 200 207916 "" "Prometheus/2.33.4" 2024-04-26T16:00:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:15 smithi064 ceph-mon[22742]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:17.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:17 smithi059 ceph-mon[21819]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:17.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:17 smithi059 ceph-mon[26536]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:17 smithi064 ceph-mon[22742]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:19.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:00:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:00:19.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:19 smithi059 ceph-mon[21819]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:19.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:19 smithi059 ceph-mon[26536]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:19.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:19 smithi064 ceph-mon[22742]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:20 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:20.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:20 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:20 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:21 smithi059 ceph-mon[21819]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:21 smithi059 ceph-mon[26536]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:21.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:21 smithi064 ceph-mon[22742]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:23.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:23 smithi064 ceph-mon[22742]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:23.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:23 smithi059 ceph-mon[26536]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:23.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:23 smithi059 ceph-mon[21819]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:25.246 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:25 smithi059 ceph-mon[21819]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:25.247 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:00:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:24] "GET /metrics HTTP/1.1" 200 207909 "" "Prometheus/2.33.4" 2024-04-26T16:00:25.247 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:25 smithi059 ceph-mon[26536]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:25.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:25 smithi064 ceph-mon[22742]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:27.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:27 smithi059 ceph-mon[21819]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:27 smithi059 ceph-mon[26536]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:27 smithi064 ceph-mon[22742]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:29.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:00:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:00:29.507 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:29 smithi059 ceph-mon[26536]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:29.507 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:29 smithi059 ceph-mon[21819]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:29.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:29 smithi064 ceph-mon[22742]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:30.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:30 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:30.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:30 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:30.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:30 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:31.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:31 smithi059 ceph-mon[26536]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:31.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:31 smithi059 ceph-mon[21819]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:31.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:31 smithi064 ceph-mon[22742]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:33 smithi064 ceph-mon[22742]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:33.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:33 smithi059 ceph-mon[26536]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:33 smithi059 ceph-mon[21819]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:35.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:35 smithi059 ceph-mon[26536]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:35.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:35 smithi059 ceph-mon[21819]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:35.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:00:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:34] "GET /metrics HTTP/1.1" 200 207909 "" "Prometheus/2.33.4" 2024-04-26T16:00:35.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:35 smithi064 ceph-mon[22742]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:37.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:37 smithi059 ceph-mon[21819]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:37 smithi059 ceph-mon[26536]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:37.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:37 smithi064 ceph-mon[22742]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:39.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:00:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:00:39.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:39 smithi059 ceph-mon[21819]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:39.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:39 smithi059 ceph-mon[26536]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:39.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:39 smithi064 ceph-mon[22742]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:40.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:40 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:40.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:40 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:40 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:41.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:41 smithi059 ceph-mon[21819]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:41 smithi059 ceph-mon[26536]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:41.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:41 smithi064 ceph-mon[22742]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:43.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:43 smithi064 ceph-mon[22742]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:43.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:43 smithi059 ceph-mon[21819]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:43.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:43 smithi059 ceph-mon[26536]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:45.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:45 smithi059 ceph-mon[21819]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:45.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:45 smithi059 ceph-mon[26536]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:45.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:00:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:44] "GET /metrics HTTP/1.1" 200 207875 "" "Prometheus/2.33.4" 2024-04-26T16:00:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:45 smithi064 ceph-mon[22742]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:47.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:47 smithi059 ceph-mon[21819]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:47.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:47 smithi059 ceph-mon[26536]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:47.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:47 smithi064 ceph-mon[22742]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:49.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:00:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:00:49.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:49 smithi059 ceph-mon[26536]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:49.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:49 smithi059 ceph-mon[21819]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:49.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:49 smithi064 ceph-mon[22742]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:50.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:50 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:50.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:50 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:50.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:50 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:00:51.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:51 smithi059 ceph-mon[26536]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:51 smithi059 ceph-mon[21819]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:51.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:51 smithi064 ceph-mon[22742]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:53.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:53 smithi064 ceph-mon[22742]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:53.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:53 smithi059 ceph-mon[26536]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:53 smithi059 ceph-mon[21819]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:55.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:55 smithi059 ceph-mon[26536]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:55.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:55 smithi059 ceph-mon[21819]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:55.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:00:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:54] "GET /metrics HTTP/1.1" 200 207876 "" "Prometheus/2.33.4" 2024-04-26T16:00:55.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:55 smithi064 ceph-mon[22742]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:00:57.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:57 smithi059 ceph-mon[26536]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:57 smithi059 ceph-mon[21819]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:57 smithi064 ceph-mon[22742]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:58.965 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-26T16:00:59.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:00:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:00:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:00:59.503 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:00:59 smithi059 ceph-mon[21819]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:59.504 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:00:59 smithi059 ceph-mon[26536]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:00:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:00:59 smithi064 ceph-mon[22742]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:00.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:00 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:00.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:00 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:00.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:00 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:01.252 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:01 smithi059 ceph-mon[21819]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:01.252 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:01 smithi059 ceph-mon[26536]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:01 smithi064 ceph-mon[22742]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:01.698 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-26T16:01:03.213 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:03 smithi059 ceph-mon[26536]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:03.213 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:03 smithi059 ceph-mon[21819]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:03.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:03 smithi064 ceph-mon[22742]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:04.154 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-26T16:01:05.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:01:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:04] "GET /metrics HTTP/1.1" 200 207876 "" "Prometheus/2.33.4" 2024-04-26T16:01:05.392 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:05 smithi059 ceph-mon[21819]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:05.392 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:05 smithi059 ceph-mon[26536]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:05 smithi064 ceph-mon[22742]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:06.768 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:01:07.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:07 smithi059 ceph-mon[26536]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:07.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:07 smithi059 ceph-mon[21819]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:07.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:07 smithi064 ceph-mon[22742]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:08.310 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:01:08.310 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:01:08.310 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:01:08.310 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:01:08.310 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:01:08.310 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:01:08.311 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:01:08.311 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:01:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:01:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:01:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:01:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:01:08.734 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:01:08.734 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (3m) 2m ago 3m 17.3M - ba2b418f427c 3d1458365b22 2024-04-26T16:01:08.734 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (3m) 2m ago 3m 49.0M - 8.3.5 dad864ee21e9 7a86c62f40c2 2024-04-26T16:01:08.734 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 running (3m) 2m ago 3m 63.6M - 3.5 e1d6a67b021e 1bab92eda9da 2024-04-26T16:01:08.734 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443 running (8m) 2m ago 8m 410M - 17.2.0 e1d6a67b021e 06cc1766b70c 2024-04-26T16:01:08.734 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:9283 running (10m) 2m ago 9m 453M - 17.2.0 e1d6a67b021e 62b620913a03 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (10m) 2m ago 10m 46.3M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (8m) 2m ago 8m 41.0M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (8m) 2m ago 8m 44.7M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (4m) 2m ago 4m 17.0M - 1dbe0e931976 ae5b4f0c81d4 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (4m) 2m ago 4m 13.8M - 1dbe0e931976 40b9dddd8b95 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (8m) 2m ago 8m 47.0M 4017M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (7m) 2m ago 7m 47.1M 4017M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (7m) 2m ago 7m 45.9M 4017M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (6m) 2m ago 6m 45.1M 4017M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (6m) 2m ago 6m 46.4M 4272M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (6m) 2m ago 6m 45.6M 4272M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (5m) 2m ago 5m 42.5M 4272M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (5m) 2m ago 5m 45.2M 4272M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:01:08.735 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (3m) 2m ago 4m 46.0M - 514e6a882f6e 316a9bec1ec5 2024-04-26T16:01:08.736 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (3m) 2m ago 3m 68.6M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:01:08.736 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (3m) 2m ago 3m 68.7M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:01:09.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:01:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:01:09.269 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions' 2024-04-26T16:01:09.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:09 smithi059 ceph-mon[21819]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:09.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:09 smithi059 ceph-mon[26536]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:09 smithi064 ceph-mon[22742]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:10.541 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:10 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:10.541 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:10 smithi059 ceph-mon[26536]: from='client.14976 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:10.541 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:10 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:10.541 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:10 smithi059 ceph-mon[21819]: from='client.14976 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:10.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:10 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:10.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:10 smithi064 ceph-mon[22742]: from='client.14976 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:11.130 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:01:11.130 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:01:11.130 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:01:11.130 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:01:11.130 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:01:11.130 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:01:11.130 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:01:11.130 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:01:11.130 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:01:11.131 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:01:11.131 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:01:11.131 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:01:11.131 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:01:11.131 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:01:11.131 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:01:11.131 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-26T16:01:11.131 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:01:11.131 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:01:11.442 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:11 smithi059 ceph-mon[21819]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:11.442 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:11 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1781316883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:01:11.442 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:11 smithi059 ceph-mon[26536]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:11.442 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:11 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1781316883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:01:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:11 smithi064 ceph-mon[22742]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:11 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1781316883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:01:11.741 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph -s' 2024-04-26T16:01:13.293 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:13 smithi059 ceph-mon[21819]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:13.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:13 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:01:13.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:13 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:01:13.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:13 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:01:13.294 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:13 smithi059 ceph-mon[26536]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:13.294 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:13 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:01:13.294 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:13 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:01:13.294 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:13 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:01:13.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:13 smithi064 ceph-mon[22742]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:13 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:01:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:13 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:01:13.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:13 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:01:13.758 INFO:teuthology.orchestra.run.smithi059.stdout: cluster: 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: id: b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: health: HEALTH_OK 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: services: 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: mgr: y(active, since 5m), standbys: x 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: data: 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: pools: 6 pools, 161 pgs 2024-04-26T16:01:13.759 INFO:teuthology.orchestra.run.smithi059.stdout: objects: 209 objects, 585 KiB 2024-04-26T16:01:13.760 INFO:teuthology.orchestra.run.smithi059.stdout: usage: 67 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:01:13.760 INFO:teuthology.orchestra.run.smithi059.stdout: pgs: 161 active+clean 2024-04-26T16:01:13.760 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:01:13.760 INFO:teuthology.orchestra.run.smithi059.stdout: io: 2024-04-26T16:01:13.760 INFO:teuthology.orchestra.run.smithi059.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-26T16:01:13.760 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:01:14.390 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:14 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:14.390 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:14 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2523593362' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:01:14.390 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:14 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:14.390 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:14 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2523593362' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:01:14.417 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ls' 2024-04-26T16:01:14.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:14 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:14.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:14 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2523593362' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:01:15.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:15 smithi059 ceph-mon[26536]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:15.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:15 smithi059 ceph-mon[21819]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:15.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:01:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:14] "GET /metrics HTTP/1.1" 200 207872 "" "Prometheus/2.33.4" 2024-04-26T16:01:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:15 smithi064 ceph-mon[22742]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:16.410 INFO:teuthology.orchestra.run.smithi059.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-26T16:01:16.410 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi059=a;count:1 2024-04-26T16:01:16.410 INFO:teuthology.orchestra.run.smithi059.stdout:grafana ?:3000 1/1 3m ago 5m smithi064=a;count:1 2024-04-26T16:01:16.410 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-04-26T16:01:16.410 INFO:teuthology.orchestra.run.smithi059.stdout:mgr 2/2 3m ago 8m smithi059=y;smithi064=x;count:2 2024-04-26T16:01:16.411 INFO:teuthology.orchestra.run.smithi059.stdout:mon 3/3 3m ago 8m smithi059:172.21.15.59=a;smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c;smithi064:172.21.15.64=b;count:3 2024-04-26T16:01:16.411 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi059=a;smithi064=b;count:2 2024-04-26T16:01:16.411 INFO:teuthology.orchestra.run.smithi059.stdout:osd 8 3m ago - 2024-04-26T16:01:16.411 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus ?:9095 1/1 3m ago 5m smithi064=a;count:1 2024-04-26T16:01:16.411 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-04-26T16:01:17.107 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-26T16:01:17.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:17 smithi059 ceph-mon[21819]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:17.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:17 smithi059 ceph-mon[26536]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:17 smithi064 ceph-mon[22742]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:18.420 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:18 smithi059 ceph-mon[21819]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:18.420 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:18 smithi059 ceph-mon[26536]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:18.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:18 smithi064 ceph-mon[22742]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:19.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:01:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:01:19.542 INFO:teuthology.orchestra.run.smithi059.stdout:Scheduled to redeploy mgr.x on host 'smithi064' 2024-04-26T16:01:19.543 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:19 smithi059 ceph-mon[21819]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:19.543 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:19 smithi059 ceph-mon[26536]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:19.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:19 smithi064 ceph-mon[22742]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:20.319 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps --refresh' 2024-04-26T16:01:20.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:20 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:20 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3326908792' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T16:01:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:20 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:20 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:01:20.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:20 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:20.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:20 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3326908792' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T16:01:20.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:20 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:20.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:20 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:01:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:20 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:20 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3326908792' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T16:01:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:20 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:20 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:01:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:21 smithi059 ceph-mon[21819]: from='client.24913 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:21 smithi059 ceph-mon[21819]: Schedule redeploy daemon mgr.x 2024-04-26T16:01:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:21 smithi059 ceph-mon[21819]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:21 smithi059 ceph-mon[26536]: from='client.24913 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:21 smithi059 ceph-mon[26536]: Schedule redeploy daemon mgr.x 2024-04-26T16:01:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:21 smithi059 ceph-mon[26536]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:21.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:21 smithi064 ceph-mon[22742]: from='client.24913 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:21 smithi064 ceph-mon[22742]: Schedule redeploy daemon mgr.x 2024-04-26T16:01:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:21 smithi064 ceph-mon[22742]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:22.450 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:01:22.450 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (3m) 3m ago 4m 17.3M - ba2b418f427c 3d1458365b22 2024-04-26T16:01:22.450 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (3m) 3m ago 3m 49.0M - 8.3.5 dad864ee21e9 7a86c62f40c2 2024-04-26T16:01:22.450 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 running (3m) 3m ago 3m 63.6M - 3.5 e1d6a67b021e 1bab92eda9da 2024-04-26T16:01:22.450 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e 06cc1766b70c 2024-04-26T16:01:22.450 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e 62b620913a03 2024-04-26T16:01:22.450 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (10m) 3m ago 10m 46.3M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:01:22.450 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (8m) 3m ago 8m 41.0M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (9m) 3m ago 9m 44.7M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (4m) 3m ago 4m 17.0M - 1dbe0e931976 ae5b4f0c81d4 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (4m) 3m ago 4m 13.8M - 1dbe0e931976 40b9dddd8b95 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (8m) 3m ago 8m 47.0M 4017M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (7m) 3m ago 7m 47.1M 4017M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (7m) 3m ago 7m 45.9M 4017M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (7m) 3m ago 7m 45.1M 4017M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (6m) 3m ago 6m 46.4M 4272M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (6m) 3m ago 6m 45.6M 4272M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (5m) 3m ago 5m 42.5M 4272M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (5m) 3m ago 5m 45.2M 4272M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (3m) 3m ago 4m 46.0M - 514e6a882f6e 316a9bec1ec5 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (3m) 3m ago 3m 68.6M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:01:22.451 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (3m) 3m ago 3m 68.7M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:01:23.162 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'sleep 180' 2024-04-26T16:01:23.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:23 smithi064 ceph-mon[22742]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:23.540 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:23 smithi059 ceph-mon[21819]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:23.540 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:23 smithi059 ceph-mon[26536]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:24.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:24 smithi064 ceph-mon[22742]: from='client.24919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:24.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:24 smithi059 ceph-mon[26536]: from='client.24919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:24 smithi059 ceph-mon[21819]: from='client.24919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:01:25.298 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:25 smithi059 ceph-mon[21819]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:25.298 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:01:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:24] "GET /metrics HTTP/1.1" 200 207862 "" "Prometheus/2.33.4" 2024-04-26T16:01:25.299 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:25 smithi059 ceph-mon[26536]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:25.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:25 smithi064 ceph-mon[22742]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:27.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:27 smithi064 ceph-mon[22742]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:27.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:27 smithi059 ceph-mon[21819]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:27 smithi059 ceph-mon[26536]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:29.172 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:01:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:01:29.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:29 smithi064 ceph-mon[22742]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:29.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:29 smithi059 ceph-mon[21819]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:29 smithi059 ceph-mon[26536]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:30.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:30 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:30.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:30 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:30.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:30 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:31.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:31 smithi064 ceph-mon[22742]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:31.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:31 smithi059 ceph-mon[26536]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:31.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:31 smithi059 ceph-mon[21819]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:33 smithi064 ceph-mon[22742]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:33.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:33 smithi059 ceph-mon[21819]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:33 smithi059 ceph-mon[26536]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:35.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:35 smithi059 ceph-mon[26536]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:35.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:35 smithi059 ceph-mon[21819]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:35.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:01:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:34] "GET /metrics HTTP/1.1" 200 207862 "" "Prometheus/2.33.4" 2024-04-26T16:01:35.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:35 smithi064 ceph-mon[22742]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:37.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:37 smithi059 ceph-mon[21819]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:37 smithi059 ceph-mon[26536]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:37.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:37 smithi064 ceph-mon[22742]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:39.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:01:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:01:39.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:39 smithi059 ceph-mon[26536]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:39.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:39 smithi059 ceph-mon[21819]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:39.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:39 smithi064 ceph-mon[22742]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:40.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:40 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:40.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:40 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:40 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:41.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:41 smithi059 ceph-mon[26536]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:41 smithi059 ceph-mon[21819]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:41.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:41 smithi064 ceph-mon[22742]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:43.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:43 smithi064 ceph-mon[22742]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:43.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:43 smithi059 ceph-mon[21819]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:43.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:43 smithi059 ceph-mon[26536]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:45.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:45 smithi059 ceph-mon[21819]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:45.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:01:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:44] "GET /metrics HTTP/1.1" 200 207863 "" "Prometheus/2.33.4" 2024-04-26T16:01:45.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:45 smithi059 ceph-mon[26536]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:45 smithi064 ceph-mon[22742]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:47 smithi059 ceph-mon[26536]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:47.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:47 smithi059 ceph-mon[21819]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:47.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:47 smithi064 ceph-mon[22742]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:49.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:01:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:01:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:48 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:48 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:01:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:48 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:01:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:48 smithi064 ceph-mon[22742]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:48 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:49.296 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:48 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:49.297 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:48 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:01:49.297 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:48 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:01:49.297 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:48 smithi059 ceph-mon[26536]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:49.297 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:48 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:49.297 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:48 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:49.297 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:48 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:01:49.297 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:48 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:01:49.297 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:48 smithi059 ceph-mon[21819]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:49.298 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:48 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:50.292 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:49 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:50.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:49 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:50.301 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:49 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:01:51.223 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:50 smithi064 ceph-mon[22742]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:51.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:50 smithi059 ceph-mon[26536]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:51.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:50 smithi059 ceph-mon[21819]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:01:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:01:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:01:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:01:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[21819]: Deploying daemon mgr.x on smithi064 2024-04-26T16:01:52.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[21819]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:01:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:01:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:01:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:01:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[26536]: Deploying daemon mgr.x on smithi064 2024-04-26T16:01:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:52 smithi059 ceph-mon[26536]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:52.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:52 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:52.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:52 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:52.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:52 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:01:52.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:52 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:01:52.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:52 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:01:52.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:52 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:01:52.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:52 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:01:52.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:52 smithi064 ceph-mon[22742]: Deploying daemon mgr.x on smithi064 2024-04-26T16:01:52.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:52 smithi064 ceph-mon[22742]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:53.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:01:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:01:53.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:01:53.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:01:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:01:53.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:01:55.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:55 smithi059 ceph-mon[26536]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:55.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:55 smithi059 ceph-mon[21819]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:55.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:01:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:54] "GET /metrics HTTP/1.1" 200 207873 "" "Prometheus/2.33.4" 2024-04-26T16:01:55.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:55 smithi064 ceph-mon[22742]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:01:57.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:57 smithi059 ceph-mon[21819]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:57.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:57 smithi059 ceph-mon[26536]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:57 smithi064 ceph-mon[22742]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:59.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:01:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:01:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:01:59.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:01:59 smithi059 ceph-mon[21819]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:59.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:01:59 smithi059 ceph-mon[26536]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:01:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:01:59 smithi064 ceph-mon[22742]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:00.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:00 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:00 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:00 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:01.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:01 smithi059 ceph-mon[26536]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:01.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:01 smithi059 ceph-mon[21819]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:01.587 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:01 smithi064 ceph-mon[22742]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:03.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:03 smithi064 ceph-mon[22742]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:03.507 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:03 smithi059 ceph-mon[21819]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:03.507 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:03 smithi059 ceph-mon[26536]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:03.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:02:03.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:02:03.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:02:03.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:02:05.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:05 smithi059 ceph-mon[21819]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:05.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:05 smithi059 ceph-mon[26536]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:05.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:02:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:04] "GET /metrics HTTP/1.1" 200 207873 "" "Prometheus/2.33.4" 2024-04-26T16:02:05.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:05 smithi064 ceph-mon[22742]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:07.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:07 smithi059 ceph-mon[21819]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:07.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:07 smithi059 ceph-mon[26536]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:07.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:07 smithi064 ceph-mon[22742]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:02:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:02:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:02:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:02:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:02:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:02:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:02:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:02:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:02:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:02:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:02:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:02:09.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:02:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:09 smithi059 ceph-mon[21819]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:09 smithi059 ceph-mon[26536]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:09 smithi064 ceph-mon[22742]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:10.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:10 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:10.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:10 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:10.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:10 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:11.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:11 smithi059 ceph-mon[21819]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:11 smithi059 ceph-mon[26536]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:11 smithi064 ceph-mon[22742]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:13.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:13 smithi064 ceph-mon[22742]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:13.508 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:13 smithi059 ceph-mon[26536]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:13.508 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:13 smithi059 ceph-mon[21819]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:13.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:02:13.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:02:13.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:02:13.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:02:15.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:15 smithi059 ceph-mon[26536]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:15.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:15 smithi059 ceph-mon[21819]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:15.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:02:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:14] "GET /metrics HTTP/1.1" 200 207885 "" "Prometheus/2.33.4" 2024-04-26T16:02:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:15 smithi064 ceph-mon[22742]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:17.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:17 smithi059 ceph-mon[26536]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:17.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:17 smithi059 ceph-mon[21819]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:17 smithi064 ceph-mon[22742]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:19.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[23922]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:02:19.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:19 smithi059 ceph-mon[26536]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:19.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:19 smithi059 ceph-mon[21819]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:19.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:19 smithi064 ceph-mon[22742]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:20.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:20 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:20.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:20 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:20.641 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:20 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:21 smithi059 ceph-mon[21819]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:21 smithi059 ceph-mon[26536]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:21.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:21 smithi064 ceph-mon[22742]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:22.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:22 smithi064 systemd[1]: Stopping Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:02:22.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:22 smithi064 bash[42803]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-26T16:02:22.779 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:22 smithi064 podman[42813]: 2024-04-26 16:02:22.50938954 +0000 UTC m=+0.451532318 container died 06cc1766b70cc044370150ae95baae1706deecea43a03451ceb11a53fa6406c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_CLEAN=True, vendor=Red Hat, Inc., name=centos-stream, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git) 2024-04-26T16:02:23.038 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:22 smithi064 podman[42813]: 2024-04-26 16:02:22.85474838 +0000 UTC m=+0.796891145 container cleanup 06cc1766b70cc044370150ae95baae1706deecea43a03451ceb11a53fa6406c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.buildah.version=1.19.8, release=754, io.openshift.expose-services=, vcs-type=git) 2024-04-26T16:02:23.039 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:22 smithi064 bash[42813]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x 2024-04-26T16:02:23.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:23 smithi064 ceph-mon[22742]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:23.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 podman[42827]: 2024-04-26 16:02:23.039452776 +0000 UTC m=+0.521132877 container remove 06cc1766b70cc044370150ae95baae1706deecea43a03451ceb11a53fa6406c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-type=git, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, ceph=True, architecture=x86_64, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_CLEAN=True, io.openshift.expose-services=) 2024-04-26T16:02:23.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T16:02:23.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 bash[42840]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-26T16:02:23.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T16:02:23.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 systemd[1]: Stopped Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:02:23.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service: Consumed 18.086s CPU time. 2024-04-26T16:02:23.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 systemd[1]: Starting Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:02:23.508 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:23 smithi059 ceph-mon[26536]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:23.508 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:23 smithi059 ceph-mon[21819]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:23.776 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 podman[42948]: 2024-04-26 16:02:23.404837692 +0000 UTC m=+0.018073725 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:02:23.776 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 podman[42948]: 2024-04-26 16:02:23.523941269 +0000 UTC m=+0.137177283 container create 03edb3b65e457699ad80c33ae495a553ed58d5bdb46cf7c44d6150fdc4e9b623 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.vendor=CentOS) 2024-04-26T16:02:23.776 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 podman[42948]: 2024-04-26 16:02:23.71005334 +0000 UTC m=+0.323289354 container init 03edb3b65e457699ad80c33ae495a553ed58d5bdb46cf7c44d6150fdc4e9b623 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:02:23.776 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 podman[42948]: 2024-04-26 16:02:23.712665528 +0000 UTC m=+0.325901544 container start 03edb3b65e457699ad80c33ae495a553ed58d5bdb46cf7c44d6150fdc4e9b623 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T16:02:23.776 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 ceph-mgr[42967]: -- 172.21.15.64:0/2808152442 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563f972a45a0 con 0x563f97278800 2024-04-26T16:02:23.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:02:23.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:02:23.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:23.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:02:23.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:02:24.072 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 bash[42948]: 03edb3b65e457699ad80c33ae495a553ed58d5bdb46cf7c44d6150fdc4e9b623 2024-04-26T16:02:24.072 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 systemd[1]: Started Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:02:24.072 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:23.871+0000 7f8419f32200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T16:02:24.072 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:24.072+0000 7f8419f32200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:02:24.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:24.198+0000 7f8419f32200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T16:02:24.822 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:24.541+0000 7f8419f32200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T16:02:24.822 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:24.630+0000 7f8419f32200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T16:02:25.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:24] "GET /metrics HTTP/1.1" 200 207877 "" "Prometheus/2.33.4" 2024-04-26T16:02:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:02:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:02:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:02:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[26536]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:02:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:02:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:02:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:24 smithi059 ceph-mon[21819]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:25.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:25.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:25.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:02:25.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:02:25.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:02:25.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:25.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:24 smithi064 ceph-mon[22742]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:26.312 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:25.985+0000 7f8419f32200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T16:02:26.312 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:26.184+0000 7f8419f32200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T16:02:26.677 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:26.371+0000 7f8419f32200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T16:02:26.677 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:26.677+0000 7f8419f32200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T16:02:26.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:26.779+0000 7f8419f32200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T16:02:27.232 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:27 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:27.004+0000 7f8419f32200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T16:02:27.232 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:27 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:27.213+0000 7f8419f32200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:02:27.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:27 smithi059 ceph-mon[21819]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:27 smithi059 ceph-mon[26536]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:27.602 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:27 smithi064 ceph-mon[22742]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:28.132 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:27 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:27.878+0000 7f8419f32200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T16:02:28.132 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:28.117+0000 7f8419f32200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-04-26T16:02:28.681 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:28.579+0000 7f8419f32200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T16:02:28.681 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:28.681+0000 7f8419f32200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T16:02:29.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:29.068+0000 7f8419f32200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T16:02:29.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:29 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:29.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:29 smithi064 ceph-mon[22742]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:29.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:29 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:29.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:29 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:29 smithi059 ceph-mon[26536]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:29 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:29 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:29 smithi059 ceph-mon[21819]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:29 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:02:29.802 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:29.446+0000 7f8419f32200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T16:02:29.903 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:29.543+0000 7f8419f32200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T16:02:29.903 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:29.802+0000 7f8419f32200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T16:02:30.086 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:29.942+0000 7f8419f32200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T16:02:30.087 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:30 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:30.043+0000 7f8419f32200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T16:02:30.380 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:30 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:30.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:30 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:30 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:30.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:30 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:30.380+0000 7f8419f32200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T16:02:30.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:30 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:30.478+0000 7f8419f32200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T16:02:31.345 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:31 smithi064 ceph-mon[22742]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:31.345 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:31.114+0000 7f8419f32200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T16:02:31.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:31 smithi059 ceph-mon[26536]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:31.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:31 smithi059 ceph-mon[21819]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:31.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:31.345+0000 7f8419f32200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T16:02:31.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:02:31] ENGINE Bus STARTING 2024-04-26T16:02:31.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: CherryPy Checker: 2024-04-26T16:02:31.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: The Application mounted at '' has an empty config. 2024-04-26T16:02:31.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:02:31.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:02:31] ENGINE Serving on http://:::9283 2024-04-26T16:02:31.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:02:31] ENGINE Bus STARTED 2024-04-26T16:02:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:32 smithi064 ceph-mon[22742]: Standby manager daemon x restarted 2024-04-26T16:02:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:32 smithi064 ceph-mon[22742]: Standby manager daemon x started 2024-04-26T16:02:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:32 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:02:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:32 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:02:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:32 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:02:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:32 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:02:32.582 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:32.316Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:02:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[21819]: Standby manager daemon x restarted 2024-04-26T16:02:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[21819]: Standby manager daemon x started 2024-04-26T16:02:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:02:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:02:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:02:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:02:32.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[26536]: Standby manager daemon x restarted 2024-04-26T16:02:32.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[26536]: Standby manager daemon x started 2024-04-26T16:02:32.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:02:32.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:02:32.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:02:32.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:32 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:02:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:33 smithi064 ceph-mon[22742]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-26T16:02:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:33 smithi064 ceph-mon[22742]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:33.509 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:33 smithi059 ceph-mon[21819]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-26T16:02:33.510 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:33 smithi059 ceph-mon[21819]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:33.510 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:33 smithi059 ceph-mon[26536]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-26T16:02:33.510 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:33 smithi059 ceph-mon[26536]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:33.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:02:33.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:02:33.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:02:33.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:02:35.233 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:02:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:34] "GET /metrics HTTP/1.1" 200 207877 "" "Prometheus/2.33.4" 2024-04-26T16:02:35.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:35 smithi059 ceph-mon[26536]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:35 smithi059 ceph-mon[21819]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:35.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:35 smithi064 ceph-mon[22742]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:37.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:37 smithi059 ceph-mon[21819]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:37 smithi059 ceph-mon[26536]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:37.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:37 smithi064 ceph-mon[22742]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:39.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:02:39.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:39 smithi059 ceph-mon[21819]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:39.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:39 smithi059 ceph-mon[26536]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:39.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:39 smithi064 ceph-mon[22742]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:40.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:40 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:40.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:40 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:40 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:41 smithi059 ceph-mon[21819]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:41 smithi059 ceph-mon[26536]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:41.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:41 smithi064 ceph-mon[22742]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:43.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:43 smithi064 ceph-mon[22742]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:43.510 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:43 smithi059 ceph-mon[21819]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:43.510 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:43 smithi059 ceph-mon[26536]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:43.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:02:43.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:02:43.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:02:43.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:02:45.236 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:02:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:44] "GET /metrics HTTP/1.1" 200 207882 "" "Prometheus/2.33.4" 2024-04-26T16:02:45.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:45 smithi059 ceph-mon[21819]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:45.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:45 smithi059 ceph-mon[26536]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:45 smithi064 ceph-mon[22742]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:47 smithi059 ceph-mon[21819]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:47.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:47 smithi059 ceph-mon[26536]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:47.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:47 smithi064 ceph-mon[22742]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:49.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:02:49.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:49 smithi059 ceph-mon[26536]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:49.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:49 smithi059 ceph-mon[21819]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:49.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:49 smithi064 ceph-mon[22742]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:50.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:50 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:50.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:50 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:50.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:50 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:02:51.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:51 smithi059 ceph-mon[21819]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:51 smithi059 ceph-mon[26536]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:51 smithi064 ceph-mon[22742]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:53.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:53 smithi064 ceph-mon[22742]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:53.510 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:53 smithi059 ceph-mon[21819]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:53.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:53 smithi059 ceph-mon[26536]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:53.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:02:53.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:02:53.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:53.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:02:53.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:02:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:02:53.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:02:55.239 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:02:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:54] "GET /metrics HTTP/1.1" 200 207883 "" "Prometheus/2.33.4" 2024-04-26T16:02:55.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:55 smithi059 ceph-mon[21819]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:55.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:55 smithi059 ceph-mon[26536]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:55.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:55 smithi064 ceph-mon[22742]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:02:57.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:57 smithi059 ceph-mon[21819]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:57.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:57 smithi059 ceph-mon[26536]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:57 smithi064 ceph-mon[22742]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:59.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:02:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:02:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:02:59.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:02:59 smithi059 ceph-mon[26536]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:59.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:02:59 smithi059 ceph-mon[21819]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:02:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:02:59 smithi064 ceph-mon[22742]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:00.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:00 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:00.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:00 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:00 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:01.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:01 smithi059 ceph-mon[26536]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:01.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:01 smithi059 ceph-mon[21819]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:01.587 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:01 smithi064 ceph-mon[22742]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:03.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:03 smithi064 ceph-mon[22742]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:03.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:03 smithi059 ceph-mon[26536]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:03.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:03 smithi059 ceph-mon[21819]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:03.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:03:03.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:03:03.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:03:03.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:03:05.242 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:03:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:04] "GET /metrics HTTP/1.1" 200 207883 "" "Prometheus/2.33.4" 2024-04-26T16:03:05.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:05 smithi059 ceph-mon[21819]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:05.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:05 smithi059 ceph-mon[26536]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:05.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:05 smithi064 ceph-mon[22742]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:07.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:07 smithi059 ceph-mon[26536]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:07.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:07 smithi059 ceph-mon[21819]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:07.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:07 smithi064 ceph-mon[22742]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:08.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:03:08.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:03:08.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:03:08.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:03:08.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:03:08.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:03:08.512 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:03:08.512 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:03:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:03:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:03:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:03:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:03:09.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:03:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:03:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:09 smithi059 ceph-mon[21819]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:09 smithi059 ceph-mon[26536]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:09 smithi064 ceph-mon[22742]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:10.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:10 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:10.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:10 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:10.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:10 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:11.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:11 smithi059 ceph-mon[21819]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:11 smithi059 ceph-mon[26536]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:11 smithi064 ceph-mon[22742]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:13.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:13 smithi064 ceph-mon[22742]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:13.512 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:13 smithi059 ceph-mon[21819]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:13.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:13 smithi059 ceph-mon[26536]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:13.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:03:13.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:03:13.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:03:13.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:13.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:03:15.246 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:03:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:14] "GET /metrics HTTP/1.1" 200 207885 "" "Prometheus/2.33.4" 2024-04-26T16:03:15.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:15 smithi059 ceph-mon[21819]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:15 smithi059 ceph-mon[26536]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:15 smithi064 ceph-mon[22742]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:17.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:17 smithi059 ceph-mon[21819]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:17.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:17 smithi059 ceph-mon[26536]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:17 smithi064 ceph-mon[22742]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:19.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:03:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:03:19.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:19 smithi059 ceph-mon[21819]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:19.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:19 smithi059 ceph-mon[26536]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:19 smithi064 ceph-mon[22742]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:20.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:20 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:20.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:20 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:20 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:21 smithi059 ceph-mon[21819]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:21 smithi059 ceph-mon[26536]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:21.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:21 smithi064 ceph-mon[22742]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:23.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:23 smithi064 ceph-mon[22742]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:23.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:23 smithi059 ceph-mon[26536]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:23.512 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:23 smithi059 ceph-mon[21819]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:23.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:03:23.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:03:23.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:23.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:03:23.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:23.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:03:25.249 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:03:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:24] "GET /metrics HTTP/1.1" 200 207885 "" "Prometheus/2.33.4" 2024-04-26T16:03:25.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:25 smithi059 ceph-mon[21819]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:25.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:25 smithi059 ceph-mon[26536]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:25.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:25 smithi064 ceph-mon[22742]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:27.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:27 smithi059 ceph-mon[21819]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:27 smithi059 ceph-mon[26536]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:27 smithi064 ceph-mon[22742]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:29.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:03:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:03:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:29 smithi059 ceph-mon[21819]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:29 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:03:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:29 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:03:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:29 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:03:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:29 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:03:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:29 smithi059 ceph-mon[26536]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:29 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:03:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:29 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:03:29.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:29 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:03:29.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:29 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:03:29.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:29 smithi064 ceph-mon[22742]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:29 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:03:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:29 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:03:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:29 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:03:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:29 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:03:30.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:30 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:30.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:30 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:30.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:30 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:31.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:31 smithi059 ceph-mon[21819]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:31.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:31 smithi059 ceph-mon[26536]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:31.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:31 smithi064 ceph-mon[22742]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:33 smithi064 ceph-mon[22742]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:33.513 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:33 smithi059 ceph-mon[26536]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:33.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:33 smithi059 ceph-mon[21819]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:33.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:03:33.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:03:33.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:03:33.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:03:35.253 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:03:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:34] "GET /metrics HTTP/1.1" 200 207885 "" "Prometheus/2.33.4" 2024-04-26T16:03:35.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:35 smithi059 ceph-mon[26536]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:35 smithi059 ceph-mon[21819]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:35.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:35 smithi064 ceph-mon[22742]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:37 smithi059 ceph-mon[26536]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:37.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:37 smithi059 ceph-mon[21819]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:37.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:37 smithi064 ceph-mon[22742]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:39.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:03:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:03:39.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:39 smithi059 ceph-mon[26536]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:39.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:39 smithi059 ceph-mon[21819]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:39.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:39 smithi064 ceph-mon[22742]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:40.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:40 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:40 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:40 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:41 smithi059 ceph-mon[26536]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:41 smithi059 ceph-mon[21819]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:41.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:41 smithi064 ceph-mon[22742]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:43.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:43 smithi064 ceph-mon[22742]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:43.513 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:43 smithi059 ceph-mon[26536]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:43.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:43 smithi059 ceph-mon[21819]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:43.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:03:43.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:03:43.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:03:43.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:43.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:03:45.256 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:03:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:44] "GET /metrics HTTP/1.1" 200 207883 "" "Prometheus/2.33.4" 2024-04-26T16:03:45.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:45 smithi059 ceph-mon[21819]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:45.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:45 smithi059 ceph-mon[26536]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:45 smithi064 ceph-mon[22742]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:47.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:47 smithi059 ceph-mon[26536]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:47 smithi059 ceph-mon[21819]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:47.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:47 smithi064 ceph-mon[22742]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:49.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:03:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:03:49.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:49 smithi059 ceph-mon[21819]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:49.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:49 smithi059 ceph-mon[26536]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:49.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:49 smithi064 ceph-mon[22742]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:50.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:50 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:50.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:50 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:50.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:50 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:03:51.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:51 smithi059 ceph-mon[21819]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:51 smithi059 ceph-mon[26536]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:51.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:51 smithi064 ceph-mon[22742]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:53.514 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:53 smithi059 ceph-mon[26536]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:53.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:53 smithi059 ceph-mon[21819]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:53.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:53 smithi064 ceph-mon[22742]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:53.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:03:53.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:03:53.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:03:53.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:03:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:03:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:03:55.258 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:03:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:54] "GET /metrics HTTP/1.1" 200 207879 "" "Prometheus/2.33.4" 2024-04-26T16:03:55.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:55 smithi059 ceph-mon[26536]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:55.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:55 smithi059 ceph-mon[21819]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:55.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:55 smithi064 ceph-mon[22742]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:03:57.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:57 smithi059 ceph-mon[21819]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:57.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:57 smithi059 ceph-mon[26536]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:57 smithi064 ceph-mon[22742]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:59.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:03:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:03:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:03:59.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:03:59 smithi059 ceph-mon[21819]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:59.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:03:59 smithi059 ceph-mon[26536]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:03:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:03:59 smithi064 ceph-mon[22742]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:00.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:00 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:00.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:00 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:00 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:01.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:01 smithi059 ceph-mon[21819]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:01 smithi059 ceph-mon[26536]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:01 smithi064 ceph-mon[22742]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:03.515 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:03 smithi059 ceph-mon[26536]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:03.515 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:03 smithi059 ceph-mon[21819]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:03.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:03 smithi064 ceph-mon[22742]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:03.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:04:03.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:04:03.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:04:03.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:04:05.261 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:04] "GET /metrics HTTP/1.1" 200 207879 "" "Prometheus/2.33.4" 2024-04-26T16:04:05.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:05 smithi059 ceph-mon[26536]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:05.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:05 smithi059 ceph-mon[21819]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:05.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:05 smithi064 ceph-mon[22742]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:07.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:07 smithi059 ceph-mon[26536]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:07.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:07 smithi059 ceph-mon[21819]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:07.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:07 smithi064 ceph-mon[22742]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:08.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:08.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:08 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:08.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:08.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:08 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:08.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:08.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:08 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:08.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:08.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:08 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:08 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:08 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:09.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:04:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:09 smithi059 ceph-mon[21819]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:09 smithi059 ceph-mon[26536]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:09 smithi064 ceph-mon[22742]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:10.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:10 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:10.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:10 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:10.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:10 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:11.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:11 smithi059 ceph-mon[21819]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:11 smithi059 ceph-mon[26536]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:11 smithi064 ceph-mon[22742]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:13.516 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:13 smithi059 ceph-mon[21819]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:13.517 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:13 smithi059 ceph-mon[26536]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:13.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:13 smithi064 ceph-mon[22742]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:13.832 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:04:13.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:04:13.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:04:13.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:04:15.267 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:14] "GET /metrics HTTP/1.1" 200 207882 "" "Prometheus/2.33.4" 2024-04-26T16:04:15.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:15 smithi059 ceph-mon[26536]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:15 smithi059 ceph-mon[21819]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:15 smithi064 ceph-mon[22742]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:17.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:17 smithi059 ceph-mon[21819]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:17.609 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:17 smithi059 ceph-mon[26536]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:17 smithi064 ceph-mon[22742]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:19.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:04:19.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:19 smithi059 ceph-mon[26536]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:19.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:19 smithi059 ceph-mon[21819]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:19.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:19 smithi064 ceph-mon[22742]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:20.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:20 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:20 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:20 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:21.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:21 smithi059 ceph-mon[21819]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:21 smithi059 ceph-mon[26536]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:21.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:21 smithi064 ceph-mon[22742]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:23.517 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:23 smithi059 ceph-mon[21819]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:23.517 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:23 smithi059 ceph-mon[26536]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:23.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:23 smithi064 ceph-mon[22742]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:23.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:04:23.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:04:23.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:04:23.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:04:25.269 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:24] "GET /metrics HTTP/1.1" 200 207885 "" "Prometheus/2.33.4" 2024-04-26T16:04:25.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:25 smithi059 ceph-mon[26536]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:25.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:25 smithi059 ceph-mon[21819]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:25.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:25 smithi064 ceph-mon[22742]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:25.837 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:04:27.272 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:27 smithi059 ceph-mon[21819]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:27.567 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:27 smithi059 ceph-mon[26536]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:27 smithi064 ceph-mon[22742]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:27.791 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:04:27.791 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (6m) 2m ago 7m 25.0M - ba2b418f427c 3d1458365b22 2024-04-26T16:04:27.792 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (6m) 119s ago 6m 56.5M - 8.3.5 dad864ee21e9 7a86c62f40c2 2024-04-26T16:04:27.792 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 running (6m) 2m ago 6m 47.0M - 3.5 e1d6a67b021e 1bab92eda9da 2024-04-26T16:04:27.792 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283 running (2m) 119s ago 11m 61.1M - 19.0.0-3305-ga430f9ed 031ba2747cfd 03edb3b65e45 2024-04-26T16:04:27.792 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:9283 running (13m) 2m ago 13m 459M - 17.2.0 e1d6a67b021e 62b620913a03 2024-04-26T16:04:27.792 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (13m) 2m ago 13m 49.6M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:04:27.792 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (11m) 119s ago 11m 41.8M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:04:27.792 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (12m) 2m ago 12m 45.5M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:04:27.792 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (7m) 2m ago 7m 18.2M - 1dbe0e931976 ae5b4f0c81d4 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (7m) 119s ago 7m 17.6M - 1dbe0e931976 40b9dddd8b95 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (11m) 2m ago 11m 49.4M 4017M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (11m) 2m ago 11m 49.2M 4017M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (10m) 2m ago 10m 48.5M 4017M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (10m) 2m ago 10m 48.5M 4017M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (9m) 119s ago 9m 49.8M 4272M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (9m) 119s ago 9m 49.1M 4272M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (9m) 119s ago 9m 45.8M 4272M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (8m) 119s ago 8m 48.6M 4272M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (6m) 119s ago 8m 71.1M - 514e6a882f6e 316a9bec1ec5 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (6m) 2m ago 6m 69.6M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:04:27.793 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (6m) 119s ago 6m 70.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:04:28.389 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions' 2024-04-26T16:04:29.047 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:04:29.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:29 smithi064 ceph-mon[22742]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:04:29.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:29 smithi064 ceph-mon[22742]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:29.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:29 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:04:29.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:29 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:04:29.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:29 smithi064 ceph-mon[22742]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:04:29.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:29 smithi064 ceph-mon[22742]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:04:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[21819]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:04:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[21819]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:04:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:04:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[21819]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:04:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[21819]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:04:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[26536]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:04:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[26536]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:04:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:04:29.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[26536]: from='mgr.24352 172.21.15.59:0/212702339' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:04:29.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:29 smithi059 ceph-mon[26536]: from='mgr.24352 ' entity='mgr.y' 2024-04-26T16:04:30.499 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:04:30.499 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:04:30.500 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:04:30.501 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-04-26T16:04:30.501 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T16:04:30.501 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:04:30.501 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:04:30.501 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:30 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:30.501 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:30 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:30.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:30 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:31.079 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph -s' 2024-04-26T16:04:31.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:31 smithi059 ceph-mon[26536]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:31.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:31 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2305902761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:04:31.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:31 smithi059 ceph-mon[21819]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:31.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:31 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2305902761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:04:31.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:31 smithi064 ceph-mon[22742]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:31.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:31 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2305902761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:04:33.118 INFO:teuthology.orchestra.run.smithi059.stdout: cluster: 2024-04-26T16:04:33.118 INFO:teuthology.orchestra.run.smithi059.stdout: id: b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:04:33.118 INFO:teuthology.orchestra.run.smithi059.stdout: health: HEALTH_OK 2024-04-26T16:04:33.118 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:04:33.118 INFO:teuthology.orchestra.run.smithi059.stdout: services: 2024-04-26T16:04:33.118 INFO:teuthology.orchestra.run.smithi059.stdout: mon: 3 daemons, quorum a,c,b (age 11m) 2024-04-26T16:04:33.118 INFO:teuthology.orchestra.run.smithi059.stdout: mgr: y(active, since 8m), standbys: x 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 8m) 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: data: 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: pools: 6 pools, 161 pgs 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: objects: 209 objects, 585 KiB 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: usage: 67 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: pgs: 161 active+clean 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: io: 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-26T16:04:33.119 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:04:33.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:33 smithi059 ceph-mon[26536]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:33.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:33 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/4063993258' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:04:33.434 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:33 smithi059 ceph-mon[21819]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:33.435 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:33 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/4063993258' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:04:33.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:33 smithi064 ceph-mon[22742]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:33 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/4063993258' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:04:33.692 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:04:33.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:04:33.692 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:04:33.692 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:04:33.717 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:04:35.008 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:34] "GET /metrics HTTP/1.1" 200 207885 "" "Prometheus/2.33.4" 2024-04-26T16:04:35.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:35 smithi059 ceph-mon[26536]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:35 smithi059 ceph-mon[21819]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:35.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:35 smithi064 ceph-mon[22742]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:35.734 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_OK 2024-04-26T16:04:36.376 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-04-26T16:04:36.404 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:36 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/183633093' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:04:36.405 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:36 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/183633093' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:04:36.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:36 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/183633093' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:04:37.556 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:37 smithi059 ceph-mon[26536]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:37.556 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:37 smithi059 ceph-mon[21819]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:37.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:37 smithi064 ceph-mon[22742]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:38.296 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:04:38.821 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph mgr fail' 2024-04-26T16:04:39.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:04:39.426 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:39 smithi059 ceph-mon[21819]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:39.426 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:39 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2489147223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:04:39.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:39 smithi059 ceph-mon[26536]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:39.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:39 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2489147223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:04:39.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:39 smithi064 ceph-mon[22742]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:04:39.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:39 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2489147223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:04:40.479 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:40 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:40.479 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:40 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:40 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:41.555 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ignoring --setuser ceph since I am not root 2024-04-26T16:04:41.555 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ignoring --setgroup ceph since I am not root 2024-04-26T16:04:41.555 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-mgr[22056]: -- 172.21.15.59:0/1695320851 <== mon.1 v2:172.21.15.59:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f9563d6340 con 0x55f9563ae800 2024-04-26T16:04:41.555 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:41.474+0000 7f1bb9eb9000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T16:04:41.555 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-mon[26536]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:41.555 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1066743957' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:04:41.555 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-mon[26536]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T16:04:41.556 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-mon[21819]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:41.556 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1066743957' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:04:41.556 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-mon[21819]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T16:04:41.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:41 smithi064 ceph-mon[22742]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:04:41.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:41 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1066743957' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:04:41.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:41 smithi064 ceph-mon[22742]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T16:04:41.565 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:04:41] ENGINE Bus STOPPING 2024-04-26T16:04:41.565 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:04:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:04:41.565 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:04:41] ENGINE Bus STOPPED 2024-04-26T16:04:41.565 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:04:41] ENGINE Bus STARTING 2024-04-26T16:04:41.566 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:04:41] ENGINE Serving on http://:::9283 2024-04-26T16:04:41.566 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:04:41] ENGINE Bus STARTED 2024-04-26T16:04:41.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:41.612+0000 7f1bb9eb9000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:04:41.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:41 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:41.716+0000 7f1bb9eb9000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T16:04:41.976 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'sleep 180' 2024-04-26T16:04:42.294 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:42.055+0000 7f1bb9eb9000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T16:04:42.294 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:42.144+0000 7f1bb9eb9000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1066743957' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: mgrmap e22: x(active, starting, since 0.363509s) 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:04:42.545 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: Manager daemon x is now available 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: Queued rgw.foo for migration 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:04:42.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:42.547 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:42.547 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:42.547 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:42.547 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.547 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1066743957' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-26T16:04:42.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: mgrmap e22: x(active, starting, since 0.363509s) 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:04:42.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: Manager daemon x is now available 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: Queued rgw.foo for migration 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1066743957' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-26T16:04:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: mgrmap e22: x(active, starting, since 0.363509s) 2024-04-26T16:04:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:04:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:04:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:04:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: Manager daemon x is now available 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: Queued rgw.foo for migration 2024-04-26T16:04:42.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-26T16:04:42.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:04:42.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:04:42.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:42.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:04:42.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:42.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:04:42.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:42.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:42 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:43.318 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:42.886Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:04:43.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:43.367+0000 7f1bb9eb9000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T16:04:43.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:43.551+0000 7f1bb9eb9000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T16:04:43.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-mon[21819]: Deploying cephadm binary to smithi059 2024-04-26T16:04:43.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-mon[21819]: mgrmap e23: x(active, since 1.38248s) 2024-04-26T16:04:43.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-mon[21819]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:43.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-mon[21819]: [26/Apr/2024:16:04:42] ENGINE Bus STARTING 2024-04-26T16:04:43.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-mon[21819]: [26/Apr/2024:16:04:42] ENGINE Serving on https://172.21.15.64:7150 2024-04-26T16:04:43.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-mon[26536]: Deploying cephadm binary to smithi059 2024-04-26T16:04:43.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-mon[26536]: mgrmap e23: x(active, since 1.38248s) 2024-04-26T16:04:43.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-mon[26536]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:43.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-mon[26536]: [26/Apr/2024:16:04:42] ENGINE Bus STARTING 2024-04-26T16:04:43.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-mon[26536]: [26/Apr/2024:16:04:42] ENGINE Serving on https://172.21.15.64:7150 2024-04-26T16:04:43.584 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:04:43.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:04:43.584 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:43.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:04:43.584 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:43.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:04:43.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:43 smithi064 ceph-mon[22742]: Deploying cephadm binary to smithi059 2024-04-26T16:04:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:43 smithi064 ceph-mon[22742]: mgrmap e23: x(active, since 1.38248s) 2024-04-26T16:04:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:43 smithi064 ceph-mon[22742]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:43 smithi064 ceph-mon[22742]: [26/Apr/2024:16:04:42] ENGINE Bus STARTING 2024-04-26T16:04:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:43 smithi064 ceph-mon[22742]: [26/Apr/2024:16:04:42] ENGINE Serving on https://172.21.15.64:7150 2024-04-26T16:04:43.972 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:43.641+0000 7f1bb9eb9000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T16:04:44.319 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:43.972+0000 7f1bb9eb9000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T16:04:44.319 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:44.068+0000 7f1bb9eb9000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T16:04:44.319 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:44.167+0000 7f1bb9eb9000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T16:04:44.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:44.424+0000 7f1bb9eb9000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:04:44.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:44 smithi059 ceph-mon[21819]: [26/Apr/2024:16:04:42] ENGINE Serving on http://172.21.15.64:8765 2024-04-26T16:04:44.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:44 smithi059 ceph-mon[21819]: [26/Apr/2024:16:04:42] ENGINE Bus STARTED 2024-04-26T16:04:44.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:44 smithi059 ceph-mon[21819]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:44.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:44 smithi059 ceph-mon[26536]: [26/Apr/2024:16:04:42] ENGINE Serving on http://172.21.15.64:8765 2024-04-26T16:04:44.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:44 smithi059 ceph-mon[26536]: [26/Apr/2024:16:04:42] ENGINE Bus STARTED 2024-04-26T16:04:44.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:44 smithi059 ceph-mon[26536]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:44.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:44 smithi064 ceph-mon[22742]: [26/Apr/2024:16:04:42] ENGINE Serving on http://172.21.15.64:8765 2024-04-26T16:04:44.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:44 smithi064 ceph-mon[22742]: [26/Apr/2024:16:04:42] ENGINE Bus STARTED 2024-04-26T16:04:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:44 smithi064 ceph-mon[22742]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:45.255 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:44.913+0000 7f1bb9eb9000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T16:04:45.565 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:45 smithi059 ceph-mon[21819]: mgrmap e24: x(active, since 3s) 2024-04-26T16:04:45.565 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:45 smithi059 ceph-mon[21819]: Deploying cephadm binary to smithi064 2024-04-26T16:04:45.566 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:45.465+0000 7f1bb9eb9000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T16:04:45.566 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:45.565+0000 7f1bb9eb9000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T16:04:45.566 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:45 smithi059 ceph-mon[26536]: mgrmap e24: x(active, since 3s) 2024-04-26T16:04:45.566 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:45 smithi059 ceph-mon[26536]: Deploying cephadm binary to smithi064 2024-04-26T16:04:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:45 smithi064 ceph-mon[22742]: mgrmap e24: x(active, since 3s) 2024-04-26T16:04:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:45 smithi064 ceph-mon[22742]: Deploying cephadm binary to smithi064 2024-04-26T16:04:46.249 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:45.892+0000 7f1bb9eb9000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T16:04:46.249 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:45.996+0000 7f1bb9eb9000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T16:04:46.574 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:46.433+0000 7f1bb9eb9000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T16:04:46.574 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:46 smithi059 ceph-mon[21819]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:46.575 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:46 smithi059 ceph-mon[26536]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:46.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:46 smithi064 ceph-mon[22742]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:46.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:46.574+0000 7f1bb9eb9000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T16:04:46.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:46.673+0000 7f1bb9eb9000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T16:04:46.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:46.802+0000 7f1bb9eb9000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T16:04:47.160 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:46.898+0000 7f1bb9eb9000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T16:04:47.499 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:47 smithi059 ceph-mon[26536]: mgrmap e25: x(active, since 5s) 2024-04-26T16:04:47.500 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:47 smithi059 ceph-mon[21819]: mgrmap e25: x(active, since 5s) 2024-04-26T16:04:47.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:47 smithi064 ceph-mon[22742]: mgrmap e25: x(active, since 5s) 2024-04-26T16:04:47.831 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:47.499+0000 7f1bb9eb9000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T16:04:48.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:47.886+0000 7f1bb9eb9000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T16:04:48.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:16:04:47] ENGINE Bus STARTING 2024-04-26T16:04:48.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: CherryPy Checker: 2024-04-26T16:04:48.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: The Application mounted at '' has an empty config. 2024-04-26T16:04:48.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: 2024-04-26T16:04:48.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:16:04:47] ENGINE Serving on http://:::9283 2024-04-26T16:04:48.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: [26/Apr/2024:16:04:47] ENGINE Bus STARTED 2024-04-26T16:04:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:48 smithi064 ceph-mon[22742]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:48 smithi064 ceph-mon[22742]: Standby manager daemon y started 2024-04-26T16:04:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:48 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T16:04:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:48 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:04:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:48 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T16:04:48.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:48 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:04:48.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:48 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:48.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:48 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[21819]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[21819]: Standby manager daemon y started 2024-04-26T16:04:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T16:04:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:04:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T16:04:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:04:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[26536]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:04:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[26536]: Standby manager daemon y started 2024-04-26T16:04:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T16:04:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:04:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T16:04:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/1142792827' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:04:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:48 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:49.127 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:48] "GET /metrics HTTP/1.1" 200 34960 "" "Prometheus/2.33.4" 2024-04-26T16:04:49.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T16:04:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:04:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:49 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:49.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:49 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:50.643 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:50 smithi064 ceph-mon[22742]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:04:50.643 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:50 smithi064 ceph-mon[22742]: Adjusting osd_memory_target on smithi059 to 2993M 2024-04-26T16:04:50.644 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:50 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:50.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:50 smithi059 ceph-mon[21819]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:04:50.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:50 smithi059 ceph-mon[21819]: Adjusting osd_memory_target on smithi059 to 2993M 2024-04-26T16:04:50.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:50 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:50.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:50 smithi059 ceph-mon[26536]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:04:50.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:50 smithi059 ceph-mon[26536]: Adjusting osd_memory_target on smithi059 to 2993M 2024-04-26T16:04:50.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:50 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:51.582 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:51 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:51.136Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:04:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:51 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:51 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:51 smithi064 ceph-mon[22742]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:04:52.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:51 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:52.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:51 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:52.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:51 smithi059 ceph-mon[26536]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:04:52.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:51 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:52.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:51 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:52.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:51 smithi059 ceph-mon[21819]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:04:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: Adjusting osd_memory_target on smithi064 to 3248M 2024-04-26T16:04:53.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:53.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:04:53.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:04:53.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:04:53.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:04:53.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:04:53.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:04:53.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:04:53.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:04:53.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:53.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:53.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: Adjusting osd_memory_target on smithi064 to 3248M 2024-04-26T16:04:53.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:53.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:04:53.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:04:53.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:04:53.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:04:53.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:04:53.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:04:53.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:04:53.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:04:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T16:04:53.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: Adjusting osd_memory_target on smithi064 to 3248M 2024-04-26T16:04:53.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:53.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:04:53.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:04:53.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:04:53.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:04:53.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:04:53.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:04:53.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:04:53.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:53 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:04:53.769 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=error ts=2024-04-26T16:04:53.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:04:53.769 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:53.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.59:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.59 because it doesn't contain any IP SANs" 2024-04-26T16:04:53.769 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=warn ts=2024-04-26T16:04:53.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.64:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.64 because it doesn't contain any IP SANs" 2024-04-26T16:04:54.082 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:53 smithi059 systemd[1]: Stopping Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:04:54.083 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:53 smithi059 bash[55571]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-26T16:04:54.083 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:53 smithi059 podman[55584]: 2024-04-26 16:04:53.94934174 +0000 UTC m=+0.144126953 container died ae5b4f0c81d44fb2c410691503121ccfbc862eb81b70eb86313bc260c61f1be1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:04:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:04:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:04:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T16:04:54.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: Reconfiguring daemon node-exporter.a on smithi059 2024-04-26T16:04:54.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:54 smithi064 ceph-mon[22742]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:04:54.580 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 podman[55584]: 2024-04-26 16:04:54.29729011 +0000 UTC m=+0.492075314 container cleanup ae5b4f0c81d44fb2c410691503121ccfbc862eb81b70eb86313bc260c61f1be1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:04:54.580 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 bash[55584]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T16:04:54.580 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 bash[55609]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-26T16:04:54.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:04:54.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:04:54.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: Reconfiguring daemon node-exporter.a on smithi059 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[21819]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T16:04:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: Reconfiguring daemon node-exporter.a on smithi059 2024-04-26T16:04:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-mon[26536]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:04:54.832 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 podman[55598]: 2024-04-26 16:04:54.581538406 +0000 UTC m=+0.635767001 container remove ae5b4f0c81d44fb2c410691503121ccfbc862eb81b70eb86313bc260c61f1be1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:04:54.833 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T16:04:54.833 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T16:04:54.833 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 systemd[1]: Stopped Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:04:54.833 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.849s CPU time. 2024-04-26T16:04:54.833 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 systemd[1]: Starting Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:04:54.833 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 podman[55681]: 2024-04-26 16:04:54.722566245 +0000 UTC m=+0.017930930 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-26T16:04:55.192 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:04:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:04:55.192 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:54 smithi059 podman[55681]: 2024-04-26 16:04:54.841018424 +0000 UTC m=+0.136383139 container create 9b14cfbf9c80879eebd8b8ccfc3d1a7f6e2ba759d7c358e4c429c37e75fc6b6f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:04:55.192 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 podman[55681]: 2024-04-26 16:04:55.068410227 +0000 UTC m=+0.363774897 container init 9b14cfbf9c80879eebd8b8ccfc3d1a7f6e2ba759d7c358e4c429c37e75fc6b6f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 podman[55681]: 2024-04-26 16:04:55.071273868 +0000 UTC m=+0.366638537 container start 9b14cfbf9c80879eebd8b8ccfc3d1a7f6e2ba759d7c358e4c429c37e75fc6b6f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.074Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.074Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.075Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.075Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=arp 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-26T16:04:55.193 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-26T16:04:55.194 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-26T16:04:55.194 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=edac 2024-04-26T16:04:55.194 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-26T16:04:55.194 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-26T16:04:55.194 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-26T16:04:55.194 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=os 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-26T16:04:55.195 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=stat 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=time 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=uname 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-26T16:04:55.196 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-26T16:04:55.197 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-26T16:04:55.197 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[55696]: ts=2024-04-26T16:04:55.076Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-26T16:04:55.197 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 bash[55681]: 9b14cfbf9c80879eebd8b8ccfc3d1a7f6e2ba759d7c358e4c429c37e75fc6b6f 2024-04-26T16:04:55.582 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:04:55 smithi059 systemd[1]: Started Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:04:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:56 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:56 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:56 smithi059 ceph-mon[26536]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T16:04:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:56 smithi059 ceph-mon[26536]: Reconfiguring daemon alertmanager.a on smithi059 2024-04-26T16:04:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:56 smithi059 ceph-mon[26536]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:04:56.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:56 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:56.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:56 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:56.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:56 smithi059 ceph-mon[21819]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T16:04:56.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:56 smithi059 ceph-mon[21819]: Reconfiguring daemon alertmanager.a on smithi059 2024-04-26T16:04:56.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:56 smithi059 ceph-mon[21819]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:04:56.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:56 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:56 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:04:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:56 smithi064 ceph-mon[22742]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T16:04:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:56 smithi064 ceph-mon[22742]: Reconfiguring daemon alertmanager.a on smithi059 2024-04-26T16:04:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:56 smithi064 ceph-mon[22742]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:04:57.489 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:57 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:04:57.489 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:57 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:04:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:57 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:04:58.262 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:58 smithi059 ceph-mon[26536]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:04:58.262 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:58 smithi059 ceph-mon[21819]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:04:58.584 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:58 smithi059 systemd[1]: Stopping Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:04:58.584 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:58 smithi059 bash[55868]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-26T16:04:58.584 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:58 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[46392]: level=info ts=2024-04-26T16:04:58.397Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-26T16:04:58.584 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:58 smithi059 podman[55881]: 2024-04-26 16:04:58.418031985 +0000 UTC m=+0.116969607 container died 3d1458365b227c2c68e8f9e71085bb141a50e0075a8ad993dedb54ba2ea2918a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:04:58.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:58 smithi064 ceph-mon[22742]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:04:58.989 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:58 smithi059 podman[55881]: 2024-04-26 16:04:58.755569705 +0000 UTC m=+0.454507325 container cleanup 3d1458365b227c2c68e8f9e71085bb141a50e0075a8ad993dedb54ba2ea2918a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:04:58.989 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:58 smithi059 bash[55881]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T16:04:59.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:04:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:04:58] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-26T16:04:59.245 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:58 smithi059 podman[55895]: 2024-04-26 16:04:58.990284897 +0000 UTC m=+0.571885103 container remove 3d1458365b227c2c68e8f9e71085bb141a50e0075a8ad993dedb54ba2ea2918a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:04:59.245 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 podman[55895]: 2024-04-26 16:04:59.057951278 +0000 UTC m=+0.639551438 volume remove 606fff395b0ea7e9ad451673ef8507b09fa00ca125117dac36ac76a091003b15 2024-04-26T16:04:59.245 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 bash[55908]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-26T16:04:59.245 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T16:04:59.245 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 systemd[1]: Stopped Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:04:59.245 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.093s CPU time. 2024-04-26T16:04:59.245 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 systemd[1]: Starting Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:04:59.524 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:04:59 smithi059 ceph-mon[26536]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:59.525 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:04:59 smithi059 ceph-mon[21819]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:04:59.525 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 podman[55975]: 2024-04-26 16:04:59.276480343 +0000 UTC m=+0.019435102 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-26T16:04:59.525 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 podman[55975]: 2024-04-26 16:04:59.391898551 +0000 UTC m=+0.134853296 volume create 567f9b5dc020900898a62ee67c288f54776c0d18df9533d368c564a0b83f8ab2 2024-04-26T16:04:59.525 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 podman[55975]: 2024-04-26 16:04:59.525773866 +0000 UTC m=+0.268728611 container create ca6d4a23fcc276a368d9a3e6062baf48ad3a444e8b095c2c8a2f70d45144ec9b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:04:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:04:59 smithi064 ceph-mon[22742]: from='client.24724 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:05:00.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 podman[55975]: 2024-04-26 16:04:59.811706504 +0000 UTC m=+0.554661250 container init ca6d4a23fcc276a368d9a3e6062baf48ad3a444e8b095c2c8a2f70d45144ec9b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:05:00.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 podman[55975]: 2024-04-26 16:04:59.814752052 +0000 UTC m=+0.557706804 container start ca6d4a23fcc276a368d9a3e6062baf48ad3a444e8b095c2c8a2f70d45144ec9b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:05:00.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:04:59.825Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-26T16:05:00.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:04:59.825Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-26T16:05:00.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:04:59.826Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.59 port=9094 2024-04-26T16:05:00.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:04:59.827Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T16:05:00.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:04:59.853Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T16:05:00.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:04:59.853Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T16:05:00.084 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:04:59.856Z caller=main.go:518 msg=Listening address=:9093 2024-04-26T16:05:00.084 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:04:59.856Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-26T16:05:00.084 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 bash[55975]: ca6d4a23fcc276a368d9a3e6062baf48ad3a444e8b095c2c8a2f70d45144ec9b 2024-04-26T16:05:00.084 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:04:59 smithi059 systemd[1]: Started Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:05:00.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[26536]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:05:00.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:00.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:00.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:00.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:00.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[21819]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:05:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:00.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:00.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:00 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:00 smithi064 ceph-mon[22742]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:05:00.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:00 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:00.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:00 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:00.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:00 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:00.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:00 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:00.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:00 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:01.518 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:01 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:05:01.518 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:01 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:05:01.519 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:01 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:05:01.519 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:01 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:05:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:01 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:05:01.588 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:01 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:05:02.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:05:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:05:01.828Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000795451s 2024-04-26T16:05:02.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:02 smithi059 ceph-mon[21819]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:02.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:02 smithi059 ceph-mon[26536]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:02.605 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:02 smithi064 ceph-mon[22742]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:02.605 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:02.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.64:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.59:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:05:03.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:05:03.144+0000 7f839ea2f700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:05:03.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Creating ceph-iscsi config... 2024-04-26T16:05:03.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:05:03.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:05:03.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Traceback (most recent call last): 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: return _run_code(code, main_globals, None, 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: exec(code, run_globals) 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:05:03.438 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:05:03.439 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:05:03.439 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:05:03.439 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:05:03.439 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:03.439 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:03.439 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Traceback (most recent call last): 2024-04-26T16:05:03.439 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: return future.result(timeout) 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: return self.__get_result() 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: raise self._exception 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: use_current_daemon_image=reconfig, 2024-04-26T16:05:03.440 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Creating ceph-iscsi config... 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: Traceback (most recent call last): 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: return _run_code(code, main_globals, None, 2024-04-26T16:05:03.441 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:05:03.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: exec(code, run_globals) 2024-04-26T16:05:03.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:05:03.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:05:03.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:05:03.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:05:03.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:05:03.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:05:03.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:05:03.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:03.442 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:04 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:05:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:05:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:05:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:05:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:05:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:05:04.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:05:04.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:05:04.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:04 smithi064 ceph-mon[22742]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:04 smithi064 ceph-mon[22742]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:04.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:04 smithi064 ceph-mon[22742]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:05:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:04 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:05:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:05:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:05:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:05:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:05:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:05:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:05:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:05:04.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:05:04.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:05:04.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:05:04.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:05:04.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:04 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:05:04.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:05:04.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:05:04.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:05:04.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:05:04.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:05:04.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:05:04.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:05:04.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:05:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:05:04.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:05:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:05:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:05:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:05:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:04 smithi059 ceph-mon[21819]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:05:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:04 smithi059 ceph-mon[21819]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:04 smithi059 ceph-mon[21819]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:05:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:05:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:05:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:05:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:05:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:05:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:04 smithi059 ceph-mon[26536]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:05:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:04 smithi059 ceph-mon[26536]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:04 smithi059 ceph-mon[26536]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:05:05.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:05:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[22052]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:05:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:05:06.572 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-mon[22742]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:06.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:06 smithi059 ceph-mon[26536]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:06 smithi059 ceph-mon[21819]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:06.877 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 systemd[1]: Stopping Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:05:06.877 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 bash[46001]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.877Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.878Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.878Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.878Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.878Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.878Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.878Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.878Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.878Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.879Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T16:05:07.170 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.879Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-26T16:05:07.171 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[39628]: ts=2024-04-26T16:05:06.879Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-26T16:05:07.171 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:06 smithi064 podman[46013]: 2024-04-26 16:05:06.888185465 +0000 UTC m=+0.123296382 container died 316a9bec1ec56d52a640251a534931570f96c5f6addc08a118f6d7090469dc73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:05:07.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:07 smithi064 podman[46013]: 2024-04-26 16:05:07.269850363 +0000 UTC m=+0.504961279 container cleanup 316a9bec1ec56d52a640251a534931570f96c5f6addc08a118f6d7090469dc73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:05:07.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:07 smithi064 bash[46013]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T16:05:07.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:07 smithi064 bash[46037]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-26T16:05:07.904 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:07 smithi064 podman[46026]: 2024-04-26 16:05:07.621028521 +0000 UTC m=+0.733825665 container remove 316a9bec1ec56d52a640251a534931570f96c5f6addc08a118f6d7090469dc73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:05:07.904 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:07 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T16:05:07.904 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:07 smithi064 systemd[1]: Stopped Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:05:07.904 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:07 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.986s CPU time. 2024-04-26T16:05:07.905 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:07 smithi064 systemd[1]: Starting Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:05:07.905 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:07 smithi064 podman[46110]: 2024-04-26 16:05:07.770583746 +0000 UTC m=+0.020079522 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-26T16:05:07.905 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:07 smithi064 podman[46110]: 2024-04-26 16:05:07.905455242 +0000 UTC m=+0.154951017 container create 27cf942569e58a068ef5ec388abef93d7f29864872f361dd97acc389c94fd02a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:05:08.187 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 podman[46110]: 2024-04-26 16:05:08.116372282 +0000 UTC m=+0.365868060 container init 27cf942569e58a068ef5ec388abef93d7f29864872f361dd97acc389c94fd02a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:05:08.187 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 podman[46110]: 2024-04-26 16:05:08.119187696 +0000 UTC m=+0.368683483 container start 27cf942569e58a068ef5ec388abef93d7f29864872f361dd97acc389c94fd02a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:05:08.187 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.143Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-26T16:05:08.187 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.143Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-26T16:05:08.187 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.143Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-26T16:05:08.187 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.143Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi064 (none))" 2024-04-26T16:05:08.187 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.143Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T16:05:08.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.143Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T16:05:08.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.145Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T16:05:08.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.145Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-26T16:05:08.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.146Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-26T16:05:08.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.149Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T16:05:08.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.152Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.987904ms 2024-04-26T16:05:08.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.152Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T16:05:08.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:08 smithi059 ceph-mon[26536]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:08 smithi059 ceph-mon[21819]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:08.622 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-mon[22742]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:08.622 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 bash[46110]: 27cf942569e58a068ef5ec388abef93d7f29864872f361dd97acc389c94fd02a 2024-04-26T16:05:08.622 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 systemd[1]: Started Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:05:08.622 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.360Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-26T16:05:08.930 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:08 smithi064 systemd[1]: Stopping Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:05:08.930 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:08 smithi064 bash[46234]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-26T16:05:08.930 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.623Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-26T16:05:08.930 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.623Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-26T16:05:08.930 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.623Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=57.679µs wal_replay_duration=471.017213ms total_replay_duration=474.100932ms 2024-04-26T16:05:08.930 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.626Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T16:05:08.931 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.626Z caller=main.go:947 level=info msg="TSDB started" 2024-04-26T16:05:08.931 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.626Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T16:05:08.931 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.653Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.266607ms db_storage=840ns remote_storage=1.158µs web_handler=365ns query_engine=759ns scrape=407.769µs scrape_sd=94.478µs notify=10.052µs notify_sd=20.475µs rules=25.37378ms 2024-04-26T16:05:08.931 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:08.653Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-26T16:05:09.180 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:08 smithi064 podman[46247]: 2024-04-26 16:05:08.930763454 +0000 UTC m=+0.109126094 container died 40b9dddd8b95f8dc6b80e7ab89d42d98a28bcb233abc690337965ec81cc0b2d2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:05:09.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-mon[22742]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T16:05:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-mon[22742]: Reconfiguring daemon node-exporter.b on smithi064 2024-04-26T16:05:09.437 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 podman[46247]: 2024-04-26 16:05:09.271385705 +0000 UTC m=+0.449748351 container cleanup 40b9dddd8b95f8dc6b80e7ab89d42d98a28bcb233abc690337965ec81cc0b2d2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:05:09.437 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 bash[46247]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T16:05:09.437 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 bash[46272]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-26T16:05:09.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:09 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:09 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:09 smithi059 ceph-mon[26536]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T16:05:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:09 smithi059 ceph-mon[26536]: Reconfiguring daemon node-exporter.b on smithi064 2024-04-26T16:05:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:09 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:09 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:09 smithi059 ceph-mon[21819]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T16:05:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:09 smithi059 ceph-mon[21819]: Reconfiguring daemon node-exporter.b on smithi064 2024-04-26T16:05:09.880 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 podman[46260]: 2024-04-26 16:05:09.54229475 +0000 UTC m=+0.615446446 container remove 40b9dddd8b95f8dc6b80e7ab89d42d98a28bcb233abc690337965ec81cc0b2d2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:05:09.880 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T16:05:09.880 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T16:05:09.881 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 systemd[1]: Stopped Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:05:09.881 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 4.030s CPU time. 2024-04-26T16:05:09.881 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 systemd[1]: Starting Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:05:09.881 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 podman[46345]: 2024-04-26 16:05:09.768111005 +0000 UTC m=+0.109415190 container create fb96f1dc25f3d2ece31db77d87801cb5a6c2fa9d1341131528278f29ce2df431 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:05:09.881 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 podman[46345]: 2024-04-26 16:05:09.676709684 +0000 UTC m=+0.018013840 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-26T16:05:10.082 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:05:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:05:09.830Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003413261s 2024-04-26T16:05:10.243 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 podman[46345]: 2024-04-26 16:05:09.979729112 +0000 UTC m=+0.321033259 container init fb96f1dc25f3d2ece31db77d87801cb5a6c2fa9d1341131528278f29ce2df431 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 podman[46345]: 2024-04-26 16:05:09.982910849 +0000 UTC m=+0.324214998 container start fb96f1dc25f3d2ece31db77d87801cb5a6c2fa9d1341131528278f29ce2df431 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.986Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.986Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.987Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.987Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=arp 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-26T16:05:10.244 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=edac 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-26T16:05:10.245 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=os 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-26T16:05:10.246 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-26T16:05:10.247 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-26T16:05:10.247 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=stat 2024-04-26T16:05:10.247 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-26T16:05:10.247 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-26T16:05:10.247 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-26T16:05:10.247 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=time 2024-04-26T16:05:10.247 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-26T16:05:10.247 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=uname 2024-04-26T16:05:10.247 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-26T16:05:10.247 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-26T16:05:10.248 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-26T16:05:10.248 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-26T16:05:10.248 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[46359]: ts=2024-04-26T16:05:09.988Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-26T16:05:10.248 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:10 smithi064 bash[46345]: fb96f1dc25f3d2ece31db77d87801cb5a6c2fa9d1341131528278f29ce2df431 2024-04-26T16:05:10.248 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:05:10 smithi064 systemd[1]: Started Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:05:10.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:10 smithi059 ceph-mon[26536]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:10.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:10 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:10.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:10 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:10.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:10 smithi059 ceph-mon[21819]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:10.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:10 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:10.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:10 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:10.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:10 smithi064 ceph-mon[22742]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:10.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:10 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:10.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:10 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:11.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[26536]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[26536]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[26536]: Reconfiguring daemon grafana.a on smithi064 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[21819]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[21819]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T16:05:11.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T16:05:11.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:11 smithi059 ceph-mon[21819]: Reconfiguring daemon grafana.a on smithi064 2024-04-26T16:05:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:11 smithi064 ceph-mon[22742]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T16:05:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:11 smithi064 ceph-mon[22742]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-26T16:05:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:11 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:11 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:11.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:11 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T16:05:11.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:11 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T16:05:11.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:11 smithi064 ceph-mon[22742]: Reconfiguring daemon grafana.a on smithi064 2024-04-26T16:05:12.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:12 smithi064 ceph-mon[22742]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:05:12.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:12 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:12.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:12 smithi059 ceph-mon[26536]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:05:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:12 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:12 smithi059 ceph-mon[21819]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:05:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:12 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:13.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:12 smithi064 systemd[1]: Stopping Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:05:13.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:12 smithi064 bash[46530]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-26T16:05:13.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[38825]: t=2024-04-26T16:05:13+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-26T16:05:13.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 podman[46542]: 2024-04-26 16:05:13.115347748 +0000 UTC m=+0.107119117 container died 7a86c62f40c26f5c2353115abbf7c5d62bb6147e1a92abfab6eab65a0e9caff8 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , architecture=x86_64, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, distribution-scope=public, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.buildah.version=1.24.2, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8) 2024-04-26T16:05:13.623 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 podman[46542]: 2024-04-26 16:05:13.44627605 +0000 UTC m=+0.438047419 container cleanup 7a86c62f40c26f5c2353115abbf7c5d62bb6147e1a92abfab6eab65a0e9caff8 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., release=236.1648460182, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2) 2024-04-26T16:05:13.623 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 bash[46542]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a 2024-04-26T16:05:13.889 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 podman[46556]: 2024-04-26 16:05:13.624026004 +0000 UTC m=+0.508440963 container remove 7a86c62f40c26f5c2353115abbf7c5d62bb6147e1a92abfab6eab65a0e9caff8 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, com.redhat.component=ubi8-container, vcs-type=git, architecture=x86_64, description=Ceph Grafana Container, name=ubi8, release=236.1648460182, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=) 2024-04-26T16:05:13.889 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 bash[46568]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-26T16:05:13.889 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-26T16:05:13.889 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 systemd[1]: Stopped Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:05:13.890 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@grafana.a.service: Consumed 3.357s CPU time. 2024-04-26T16:05:13.890 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 systemd[1]: Starting Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:05:14.140 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 podman[46638]: 2024-04-26 16:05:13.792540047 +0000 UTC m=+0.017788409 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-26T16:05:14.140 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:13 smithi064 podman[46638]: 2024-04-26 16:05:13.89020734 +0000 UTC m=+0.115455731 container create 0d16158d97911e189ca131644d6ad8e7a386ce67f02da5ace5048bacc9fc0dd0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., vcs-type=git, description=Ceph Grafana Container, version=8.5, io.openshift.expose-services=) 2024-04-26T16:05:14.141 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 podman[46638]: 2024-04-26 16:05:14.109574943 +0000 UTC m=+0.334823293 container init 0d16158d97911e189ca131644d6ad8e7a386ce67f02da5ace5048bacc9fc0dd0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, description=Ceph Grafana Container, io.openshift.expose-services=, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.buildah.version=1.24.2, release=236.1648460182, version=8.5, name=ubi8, build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, vendor=Red Hat, Inc.) 2024-04-26T16:05:14.141 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 podman[46638]: 2024-04-26 16:05:14.112261488 +0000 UTC m=+0.337509840 container start 0d16158d97911e189ca131644d6ad8e7a386ce67f02da5ace5048bacc9fc0dd0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, vendor=Red Hat, Inc., release=236.1648460182, build-date=2022-03-28T10:36:18.413762, architecture=x86_64, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , io.buildah.version=1.24.2, io.openshift.expose-services=, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T16:05:14.421 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STOPPING 2024-04-26T16:05:14.421 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:05:14.421 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STOPPED 2024-04-26T16:05:14.421 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STARTING 2024-04-26T16:05:14.421 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Serving on http://:::9283 2024-04-26T16:05:14.421 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:05:14.421 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:05:14.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T16:05:14.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:05:14.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:05:14.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:05:14.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064:3000"}]: dispatch 2024-04-26T16:05:14.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-26T16:05:14.423 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="App mode production" logger=settings 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=510.989µs 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-26T16:05:14.424 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 bash[46638]: 0d16158d97911e189ca131644d6ad8e7a386ce67f02da5ace5048bacc9fc0dd0 2024-04-26T16:05:14.425 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 systemd[1]: Started Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:05:14.425 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-04-26T16:05:14.425 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-26T16:05:14.425 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-04-26T16:05:14.425 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-26T16:05:14.425 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-26T16:05:14.425 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:05:14+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-26T16:05:14.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064:3000"}]: dispatch 2024-04-26T16:05:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064:3000"}]: dispatch 2024-04-26T16:05:14.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:14 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:14.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STARTED 2024-04-26T16:05:14.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STOPPING 2024-04-26T16:05:14.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:05:14.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STOPPED 2024-04-26T16:05:14.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STARTING 2024-04-26T16:05:14.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Serving on http://:::9283 2024-04-26T16:05:14.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STARTED 2024-04-26T16:05:14.688 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STOPPING 2024-04-26T16:05:14.688 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:05:14.688 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STOPPED 2024-04-26T16:05:14.688 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STARTING 2024-04-26T16:05:14.688 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Serving on http://:::9283 2024-04-26T16:05:14.688 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:05:14] ENGINE Bus STARTED 2024-04-26T16:05:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:05:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T16:05:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:05:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:05:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:05:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064:3000"}]: dispatch 2024-04-26T16:05:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:05:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2173744458' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:05:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3422547814' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3519778281"}]: dispatch 2024-04-26T16:05:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3519778281"}]: dispatch 2024-04-26T16:05:15.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:05:15.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T16:05:15.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:05:15.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:05:15.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:05:15.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064:3000"}]: dispatch 2024-04-26T16:05:15.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:05:15.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2173744458' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:05:15.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3422547814' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3519778281"}]: dispatch 2024-04-26T16:05:15.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:15 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3519778281"}]: dispatch 2024-04-26T16:05:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:15 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:05:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:15 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T16:05:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:15 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:05:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:15 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:05:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:15 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:05:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:15 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064:3000"}]: dispatch 2024-04-26T16:05:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:15 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:05:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:15 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2173744458' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:05:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:15 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3422547814' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3519778281"}]: dispatch 2024-04-26T16:05:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:15 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3519778281"}]: dispatch 2024-04-26T16:05:16.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:16 smithi064 ceph-mon[22742]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:05:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:16 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3519778281"}]': finished 2024-04-26T16:05:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:16 smithi064 ceph-mon[22742]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T16:05:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:16 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1147826005' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1390537899"}]: dispatch 2024-04-26T16:05:16.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:16 smithi059 ceph-mon[21819]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:05:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:16 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3519778281"}]': finished 2024-04-26T16:05:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:16 smithi059 ceph-mon[21819]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T16:05:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:16 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1147826005' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1390537899"}]: dispatch 2024-04-26T16:05:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:16 smithi059 ceph-mon[26536]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:05:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:16 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3519778281"}]': finished 2024-04-26T16:05:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:16 smithi059 ceph-mon[26536]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T16:05:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:16 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1147826005' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1390537899"}]: dispatch 2024-04-26T16:05:17.630 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:17 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1147826005' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1390537899"}]': finished 2024-04-26T16:05:17.631 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:17 smithi059 ceph-mon[21819]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T16:05:17.631 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:17 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3730239837' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3141215399"}]: dispatch 2024-04-26T16:05:17.631 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:17 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1147826005' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1390537899"}]': finished 2024-04-26T16:05:17.631 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:17 smithi059 ceph-mon[26536]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T16:05:17.631 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:17 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3730239837' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3141215399"}]: dispatch 2024-04-26T16:05:17.640 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:17 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1147826005' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1390537899"}]': finished 2024-04-26T16:05:17.640 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:17 smithi064 ceph-mon[22742]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T16:05:17.640 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:17 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3730239837' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3141215399"}]: dispatch 2024-04-26T16:05:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:18 smithi064 ceph-mon[22742]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:05:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:18 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3730239837' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3141215399"}]': finished 2024-04-26T16:05:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:18 smithi064 ceph-mon[22742]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T16:05:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:18 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3672192847' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/3669944706"}]: dispatch 2024-04-26T16:05:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:18 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:18 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:18.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[26536]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3730239837' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3141215399"}]': finished 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[26536]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3672192847' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/3669944706"}]: dispatch 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[21819]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3730239837' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3141215399"}]': finished 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[21819]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3672192847' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/3669944706"}]: dispatch 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:18 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:19.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3672192847' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/3669944706"}]': finished 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2088578302' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/4008180956"}]: dispatch 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/4008180956"}]: dispatch 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:19 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:19.652 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3672192847' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/3669944706"}]': finished 2024-04-26T16:05:19.652 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T16:05:19.652 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2088578302' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/4008180956"}]: dispatch 2024-04-26T16:05:19.652 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/4008180956"}]: dispatch 2024-04-26T16:05:19.652 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3672192847' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/3669944706"}]': finished 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2088578302' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/4008180956"}]: dispatch 2024-04-26T16:05:19.653 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/4008180956"}]: dispatch 2024-04-26T16:05:19.654 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:19.654 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:19.654 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:19.654 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:05:19.654 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:19.654 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:19.654 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:05:19.654 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:19 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:20 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:05:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:20 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:05:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:20 smithi064 ceph-mon[22742]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:05:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:20 smithi064 ceph-mon[22742]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/4008180956"}]': finished 2024-04-26T16:05:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:20 smithi064 ceph-mon[22742]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T16:05:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:20 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3759155853' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/3669944706"}]: dispatch 2024-04-26T16:05:20.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:05:20.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:05:20.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[26536]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:05:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[26536]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/4008180956"}]': finished 2024-04-26T16:05:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[26536]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T16:05:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3759155853' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/3669944706"}]: dispatch 2024-04-26T16:05:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:05:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:05:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[21819]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:05:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[21819]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/4008180956"}]': finished 2024-04-26T16:05:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[21819]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T16:05:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:20 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3759155853' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/3669944706"}]: dispatch 2024-04-26T16:05:21.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:21 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3759155853' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/3669944706"}]': finished 2024-04-26T16:05:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:21 smithi064 ceph-mon[22742]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T16:05:21.713 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:21 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3759155853' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/3669944706"}]': finished 2024-04-26T16:05:21.713 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:21 smithi059 ceph-mon[21819]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T16:05:21.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:21 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3759155853' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/3669944706"}]': finished 2024-04-26T16:05:21.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:21 smithi059 ceph-mon[26536]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T16:05:22.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:22 smithi059 ceph-mon[21819]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:05:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:22 smithi059 ceph-mon[26536]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:05:22.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:22 smithi064 ceph-mon[22742]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:05:23.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:05:22] "GET /metrics HTTP/1.1" 200 37962 "" "Prometheus/2.33.4" 2024-04-26T16:05:23.888 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:05:24.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:24.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:24.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:05:24.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:05:24.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:05:24.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:05:24.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:05:24.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-mon[22742]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-04-26T16:05:24.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3757159735' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:05:24.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:23 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2877350976' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/144595486"}]: dispatch 2024-04-26T16:05:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:05:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:05:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:05:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:05:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:05:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[26536]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-04-26T16:05:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3757159735' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:05:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2877350976' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/144595486"}]: dispatch 2024-04-26T16:05:24.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:24.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:24.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:05:24.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:05:24.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:05:24.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:05:24.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:05:24.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[21819]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 1 op/s 2024-04-26T16:05:24.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3757159735' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:05:24.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:23 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2877350976' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/144595486"}]: dispatch 2024-04-26T16:05:25.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:24 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2877350976' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/144595486"}]': finished 2024-04-26T16:05:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:24 smithi059 ceph-mon[26536]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T16:05:25.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:24 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2877350976' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/144595486"}]': finished 2024-04-26T16:05:25.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:24 smithi059 ceph-mon[21819]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T16:05:25.420 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:24 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2877350976' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/144595486"}]': finished 2024-04-26T16:05:25.420 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:24 smithi064 ceph-mon[22742]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T16:05:26.247 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:25 smithi064 ceph-mon[22742]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:26.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:25 smithi059 ceph-mon[21819]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:26.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:25 smithi059 ceph-mon[26536]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:27.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:26 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:27.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:26 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:27.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:26 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:28.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:28 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:28 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:28 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:28 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:28 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:28.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:28 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:05:28.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:28 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:28 smithi064 ceph-mon[22742]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 642 B/s rd, 0 op/s 2024-04-26T16:05:28.490 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.491 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.491 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.491 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.491 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:28.491 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:05:28.491 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.491 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[21819]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 642 B/s rd, 0 op/s 2024-04-26T16:05:28.491 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.491 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.492 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.492 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.492 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:05:28.492 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:05:28.492 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:05:28.492 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:28 smithi059 ceph-mon[26536]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 642 B/s rd, 0 op/s 2024-04-26T16:05:28.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:05:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:30 smithi064 ceph-mon[22742]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:05:30.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:30 smithi059 ceph-mon[26536]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:05:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:30 smithi059 ceph-mon[21819]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:05:32.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:32 smithi064 ceph-mon[22742]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:05:32.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:32 smithi059 ceph-mon[21819]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:05:32.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:32 smithi059 ceph-mon[26536]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:05:33.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:05:32] "GET /metrics HTTP/1.1" 200 37957 "" "Prometheus/2.33.4" 2024-04-26T16:05:33.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:33 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:05:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:34 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:05:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:34 smithi064 ceph-mon[22742]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:05:34.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:34 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:05:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:34 smithi059 ceph-mon[21819]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:05:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:34 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:05:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:34 smithi059 ceph-mon[26536]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:05:36.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:36 smithi064 ceph-mon[22742]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 903 B/s rd, 0 op/s 2024-04-26T16:05:36.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:36 smithi059 ceph-mon[21819]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 903 B/s rd, 0 op/s 2024-04-26T16:05:36.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:36 smithi059 ceph-mon[26536]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 903 B/s rd, 0 op/s 2024-04-26T16:05:38.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:38 smithi059 ceph-mon[21819]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:38.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:38 smithi059 ceph-mon[26536]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:38.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:38 smithi064 ceph-mon[22742]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:38.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:38.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:05:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:40 smithi064 ceph-mon[22742]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:40.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:40 smithi059 ceph-mon[21819]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:40 smithi059 ceph-mon[26536]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:42 smithi064 ceph-mon[22742]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:42.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:42 smithi059 ceph-mon[21819]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:42.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:42.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:42 smithi059 ceph-mon[26536]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:42.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:43.169 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:05:42] "GET /metrics HTTP/1.1" 200 37962 "" "Prometheus/2.33.4" 2024-04-26T16:05:43.764 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:05:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:44 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:05:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:44 smithi064 ceph-mon[22742]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:44.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:44 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:05:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:44 smithi059 ceph-mon[21819]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:44 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:05:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:44 smithi059 ceph-mon[26536]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:46.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:46 smithi064 ceph-mon[22742]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:46.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:46 smithi059 ceph-mon[21819]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:46.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:46 smithi059 ceph-mon[26536]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:48.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:48 smithi059 ceph-mon[21819]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:48.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:48 smithi059 ceph-mon[26536]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:48.657 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:48 smithi064 ceph-mon[22742]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:48.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:48.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:05:50.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:50 smithi064 ceph-mon[22742]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:50.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:50 smithi059 ceph-mon[21819]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:50.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:50 smithi059 ceph-mon[26536]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:52.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:52 smithi064 ceph-mon[22742]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:52.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:52 smithi059 ceph-mon[21819]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:52.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:52 smithi059 ceph-mon[26536]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:53.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:05:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:05:52] "GET /metrics HTTP/1.1" 200 37962 "" "Prometheus/2.33.4" 2024-04-26T16:05:53.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:05:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:54 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:05:54.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:54 smithi064 ceph-mon[22742]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:54.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:54 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:05:54.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:54 smithi059 ceph-mon[21819]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:54 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:05:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:54 smithi059 ceph-mon[26536]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:05:56.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:56 smithi064 ceph-mon[22742]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:56.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:56 smithi059 ceph-mon[21819]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:56 smithi059 ceph-mon[26536]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:57 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:57.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:57 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:57.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:57 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:05:58.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:05:58 smithi064 ceph-mon[22742]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:58.734 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:05:58 smithi059 ceph-mon[21819]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:58.734 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:05:58 smithi059 ceph-mon[26536]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:05:58.936 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:05:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:05:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:00 smithi064 ceph-mon[22742]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:00.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:00 smithi059 ceph-mon[21819]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:00.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:00 smithi059 ceph-mon[26536]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:02.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:02 smithi064 ceph-mon[22742]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:02.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:02 smithi059 ceph-mon[21819]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:02.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:02 smithi059 ceph-mon[26536]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:03.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:06:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:06:02] "GET /metrics HTTP/1.1" 200 37959 "" "Prometheus/2.33.4" 2024-04-26T16:06:03.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:04 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:04 smithi064 ceph-mon[22742]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:04.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:04 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:04 smithi059 ceph-mon[21819]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:04 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:04 smithi059 ceph-mon[26536]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:06.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:06 smithi064 ceph-mon[22742]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:06.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:06 smithi059 ceph-mon[21819]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:06.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:06 smithi059 ceph-mon[26536]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:08.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:08 smithi059 ceph-mon[21819]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:08 smithi059 ceph-mon[26536]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:08.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:08 smithi064 ceph-mon[22742]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:08.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:10.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:10 smithi064 ceph-mon[22742]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:10.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:10 smithi059 ceph-mon[26536]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:10 smithi059 ceph-mon[21819]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:12.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:12 smithi064 ceph-mon[22742]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:12 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:12.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:12 smithi059 ceph-mon[21819]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:12 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:12.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:12 smithi059 ceph-mon[26536]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:12.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:12 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:13.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:06:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:06:12] "GET /metrics HTTP/1.1" 200 37956 "" "Prometheus/2.33.4" 2024-04-26T16:06:13.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:14.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:14 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:14 smithi064 ceph-mon[22742]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:14 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:14 smithi059 ceph-mon[21819]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:14 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:14 smithi059 ceph-mon[26536]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:16 smithi064 ceph-mon[22742]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:16 smithi059 ceph-mon[21819]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:16 smithi059 ceph-mon[26536]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:18.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:18 smithi059 ceph-mon[21819]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:18 smithi059 ceph-mon[26536]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:18.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:18 smithi064 ceph-mon[22742]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:18.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:20 smithi064 ceph-mon[22742]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:20.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:20 smithi059 ceph-mon[21819]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:20 smithi059 ceph-mon[26536]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:22.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:22 smithi064 ceph-mon[22742]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:22.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:22 smithi059 ceph-mon[21819]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:22.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:22 smithi059 ceph-mon[26536]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:23.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:06:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:06:22] "GET /metrics HTTP/1.1" 200 37956 "" "Prometheus/2.33.4" 2024-04-26T16:06:23.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:24.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:24 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:24 smithi064 ceph-mon[22742]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:24.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:24 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:24.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:24 smithi059 ceph-mon[21819]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:24 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:24 smithi059 ceph-mon[26536]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:26.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:26 smithi064 ceph-mon[22742]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:26.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:26 smithi059 ceph-mon[21819]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:26 smithi059 ceph-mon[26536]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:27 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:27.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:27 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:27.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:27 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:28.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:28 smithi059 ceph-mon[21819]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:28.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:28 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:06:28.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:28 smithi059 ceph-mon[26536]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:28.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:28 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:06:28.657 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:28 smithi064 ceph-mon[22742]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:28.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:28 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:06:28.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:28.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:30 smithi064 ceph-mon[22742]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:30.794 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:30 smithi059 ceph-mon[21819]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:30.794 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:30 smithi059 ceph-mon[26536]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:32.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:32 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:32 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:32 smithi064 ceph-mon[22742]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:32 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:32 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:32 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:32 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:32 smithi059 ceph-mon[26536]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:32.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:32 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:32 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:32 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:32 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:32 smithi059 ceph-mon[21819]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:32 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:32 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:33.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:06:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:06:32] "GET /metrics HTTP/1.1" 200 37958 "" "Prometheus/2.33.4" 2024-04-26T16:06:33.506 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:33 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:06:33.506 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:33 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:06:33.507 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:33 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:33.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:33 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:06:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:33 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:06:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:33 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:33 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:06:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:33 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:06:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:33 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:06:33.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:33 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:33.507Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:34 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:34 smithi059 ceph-mon[26536]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:34.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:34 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:34.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:34 smithi059 ceph-mon[21819]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:34 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:34 smithi064 ceph-mon[22742]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:36.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:36 smithi064 ceph-mon[22742]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:36.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:36 smithi059 ceph-mon[26536]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:36.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:36 smithi059 ceph-mon[21819]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:38.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:38 smithi059 ceph-mon[26536]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:38.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:38 smithi059 ceph-mon[21819]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:38.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:38 smithi064 ceph-mon[22742]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:38.936 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:40 smithi064 ceph-mon[22742]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:40.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:40 smithi059 ceph-mon[26536]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:40.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:40 smithi059 ceph-mon[21819]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:42.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:42 smithi064 ceph-mon[22742]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:42.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:42 smithi059 ceph-mon[26536]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:42.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:42.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:42 smithi059 ceph-mon[21819]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:42.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:43.101 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:06:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:06:42] "GET /metrics HTTP/1.1" 200 37957 "" "Prometheus/2.33.4" 2024-04-26T16:06:43.936 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:44.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:44 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:44 smithi064 ceph-mon[22742]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:44 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:44 smithi059 ceph-mon[21819]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:44 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:44 smithi059 ceph-mon[26536]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:46.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:46 smithi064 ceph-mon[22742]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:46.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:46 smithi059 ceph-mon[21819]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:46.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:46 smithi059 ceph-mon[26536]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:48.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:48 smithi059 ceph-mon[21819]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:48.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:48 smithi059 ceph-mon[26536]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:48.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:48 smithi064 ceph-mon[22742]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:48.936 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:50.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:50 smithi064 ceph-mon[22742]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:50.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:50 smithi059 ceph-mon[21819]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:50.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:50 smithi059 ceph-mon[26536]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:52.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:52 smithi064 ceph-mon[22742]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:52.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:52 smithi059 ceph-mon[21819]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:52.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:52 smithi059 ceph-mon[26536]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:53.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:06:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:06:52] "GET /metrics HTTP/1.1" 200 37957 "" "Prometheus/2.33.4" 2024-04-26T16:06:53.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:54 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:54 smithi064 ceph-mon[22742]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:54.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:54 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:54.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:54 smithi059 ceph-mon[21819]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:54 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:06:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:54 smithi059 ceph-mon[26536]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:56 smithi064 ceph-mon[22742]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:56.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:56 smithi059 ceph-mon[21819]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:56 smithi059 ceph-mon[26536]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:06:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:57 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:57.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:57 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:57.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:57 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:06:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:06:58 smithi064 ceph-mon[22742]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:58.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:06:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:06:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:06:58.734 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:06:58 smithi059 ceph-mon[21819]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:06:58.735 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:06:58 smithi059 ceph-mon[26536]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:00 smithi064 ceph-mon[22742]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:00.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:00 smithi059 ceph-mon[21819]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:00.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:00 smithi059 ceph-mon[26536]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:02.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:02 smithi064 ceph-mon[22742]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:02.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:02 smithi059 ceph-mon[21819]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:02.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:02 smithi059 ceph-mon[26536]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:03.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:07:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:07:02] "GET /metrics HTTP/1.1" 200 37961 "" "Prometheus/2.33.4" 2024-04-26T16:07:03.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:04 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:04 smithi064 ceph-mon[22742]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:04 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:04 smithi059 ceph-mon[21819]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:04 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:04 smithi059 ceph-mon[26536]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:06.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:06 smithi064 ceph-mon[22742]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:06 smithi059 ceph-mon[21819]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:06.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:06 smithi059 ceph-mon[26536]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:08.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:08 smithi059 ceph-mon[21819]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:08 smithi059 ceph-mon[26536]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:08.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:08 smithi064 ceph-mon[22742]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:08.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:10.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:10 smithi064 ceph-mon[22742]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:10.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:10 smithi059 ceph-mon[26536]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:10 smithi059 ceph-mon[21819]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:12 smithi064 ceph-mon[22742]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:12 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:12.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:12 smithi059 ceph-mon[26536]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:12.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:12 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:12 smithi059 ceph-mon[21819]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:12 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:13.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:07:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:07:12] "GET /metrics HTTP/1.1" 200 37963 "" "Prometheus/2.33.4" 2024-04-26T16:07:13.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:14 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:14 smithi064 ceph-mon[22742]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:14 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:14 smithi059 ceph-mon[21819]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:14 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:14 smithi059 ceph-mon[26536]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:16 smithi064 ceph-mon[22742]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:16.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:16 smithi059 ceph-mon[26536]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:16.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:16 smithi059 ceph-mon[21819]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:18.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:18 smithi059 ceph-mon[21819]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:18 smithi059 ceph-mon[26536]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:18.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:18 smithi064 ceph-mon[22742]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:18.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:20 smithi064 ceph-mon[22742]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:20.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:20 smithi059 ceph-mon[21819]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:20 smithi059 ceph-mon[26536]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:22.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:22 smithi064 ceph-mon[22742]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:22.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:22 smithi059 ceph-mon[21819]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:22.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:22 smithi059 ceph-mon[26536]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:23.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:07:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:07:22] "GET /metrics HTTP/1.1" 200 37963 "" "Prometheus/2.33.4" 2024-04-26T16:07:23.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:24.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:24 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:24 smithi064 ceph-mon[22742]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:24.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:24 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:24.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:24 smithi059 ceph-mon[21819]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:24 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:24 smithi059 ceph-mon[26536]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:26.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:26 smithi064 ceph-mon[22742]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:26.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:26 smithi059 ceph-mon[21819]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:26 smithi059 ceph-mon[26536]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:27.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:27 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:27 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:27.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:27 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:28.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:28 smithi064 ceph-mon[22742]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:28.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:28.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:28.734 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:28 smithi059 ceph-mon[21819]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:28.734 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:28 smithi059 ceph-mon[26536]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:30.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:30 smithi064 ceph-mon[22742]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:07:30.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:30 smithi059 ceph-mon[21819]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:07:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:30 smithi059 ceph-mon[26536]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:07:32.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:32 smithi064 ceph-mon[22742]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:07:32.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:32 smithi059 ceph-mon[21819]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:07:32.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:32 smithi059 ceph-mon[26536]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:07:33.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:07:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:07:32] "GET /metrics HTTP/1.1" 200 37967 "" "Prometheus/2.33.4" 2024-04-26T16:07:33.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:33 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:07:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:33 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:07:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:33 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:07:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:33 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:33 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:07:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:33 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:07:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:33 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:07:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:33 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:33.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:33 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:07:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:33 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:07:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:33 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:07:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:33 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:33.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:33 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:34 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:34 smithi064 ceph-mon[22742]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:34.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:34 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:34.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:34 smithi059 ceph-mon[26536]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:34 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:34 smithi059 ceph-mon[21819]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:36.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:36 smithi059 ceph-mon[26536]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:07:36.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:36 smithi059 ceph-mon[21819]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:07:36.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:36 smithi064 ceph-mon[22742]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:07:38.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:38 smithi059 ceph-mon[26536]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:38.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:38 smithi059 ceph-mon[21819]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:38.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:38 smithi064 ceph-mon[22742]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:38.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:40 smithi064 ceph-mon[22742]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:40.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:40 smithi059 ceph-mon[21819]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:40 smithi059 ceph-mon[26536]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:42.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:42 smithi064 ceph-mon[22742]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:42 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:42.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:42 smithi059 ceph-mon[21819]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:42.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:42 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:42.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:42 smithi059 ceph-mon[26536]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:42.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:42 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:43.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:07:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:07:42] "GET /metrics HTTP/1.1" 200 37963 "" "Prometheus/2.33.4" 2024-04-26T16:07:43.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:44.381 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-26T16:07:44.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:44 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:44 smithi064 ceph-mon[22742]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:44.702 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:44 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:44.703 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:44 smithi059 ceph-mon[26536]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:44.703 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:44 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:44.703 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:44 smithi059 ceph-mon[21819]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:46.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:46 smithi064 ceph-mon[22742]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:46.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:46 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/75119345' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T16:07:46.794 INFO:teuthology.orchestra.run.smithi059.stdout:Scheduled to redeploy mgr.y on host 'smithi059' 2024-04-26T16:07:46.795 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:46 smithi059 ceph-mon[26536]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:46.795 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:46 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/75119345' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T16:07:46.795 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:46 smithi059 ceph-mon[21819]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:46.795 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:46 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/75119345' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T16:07:47.398 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps --refresh' 2024-04-26T16:07:48.017 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[26536]: from='client.15183 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:07:48.017 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:48.017 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[26536]: Schedule redeploy daemon mgr.y 2024-04-26T16:07:48.017 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:48.017 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:48.017 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:07:48.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[21819]: from='client.15183 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:07:48.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:48.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[21819]: Schedule redeploy daemon mgr.y 2024-04-26T16:07:48.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:48.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:48.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:47 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:07:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:47 smithi064 ceph-mon[22742]: from='client.15183 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:07:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:47 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:47 smithi064 ceph-mon[22742]: Schedule redeploy daemon mgr.y 2024-04-26T16:07:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:47 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:47 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:47 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:07:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:48 smithi064 ceph-mon[22742]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:48.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:48.952 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:48 smithi059 ceph-mon[26536]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:48.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:48 smithi059 ceph-mon[21819]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:50.275 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:07:50.275 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (2m) 79s ago 10m 16.8M - 0.23.0 ba2b418f427c ca6d4a23fcc2 2024-04-26T16:07:50.275 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (2m) 78s ago 10m 47.0M - 8.3.5 dad864ee21e9 0d16158d9791 2024-04-26T16:07:50.275 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 running (2m) 79s ago 9m 50.5M - 3.5 e1d6a67b021e 65a32364fe46 2024-04-26T16:07:50.275 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283 running (5m) 78s ago 15m 521M - 19.0.0-3305-ga430f9ed 031ba2747cfd 03edb3b65e45 2024-04-26T16:07:50.275 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:9283 running (16m) 79s ago 16m 411M - 17.2.0 e1d6a67b021e 62b620913a03 2024-04-26T16:07:50.275 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (16m) 79s ago 16m 54.9M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:07:50.275 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (15m) 78s ago 15m 48.7M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (15m) 79s ago 15m 50.4M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (2m) 79s ago 11m 16.5M - 1.3.1 1dbe0e931976 9b14cfbf9c80 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (2m) 78s ago 11m 18.5M - 1.3.1 1dbe0e931976 fb96f1dc25f3 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (14m) 79s ago 14m 51.8M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (14m) 79s ago 14m 51.3M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (13m) 79s ago 13m 49.8M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (13m) 79s ago 13m 50.7M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (13m) 78s ago 13m 51.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (12m) 78s ago 12m 51.6M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (12m) 78s ago 12m 48.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (12m) 78s ago 12m 50.8M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (2m) 78s ago 11m 65.9M - 2.33.4 514e6a882f6e 27cf942569e5 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (9m) 79s ago 9m 71.2M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:07:50.276 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (9m) 78s ago 9m 71.5M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:07:51.067 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:50 smithi059 ceph-mon[21819]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:51.067 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:50 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:51.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:50 smithi059 ceph-mon[26536]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:51.068 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:50 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:51.091 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'sleep 180' 2024-04-26T16:07:51.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:50 smithi064 ceph-mon[22742]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:51.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:50 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:52.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:51 smithi059 ceph-mon[26536]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:07:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:51 smithi059 ceph-mon[21819]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:07:52.098 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:51 smithi064 ceph-mon[22742]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:07:53.045 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:52 smithi064 ceph-mon[22742]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:53.045 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:07:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:07:52] "GET /metrics HTTP/1.1" 200 37963 "" "Prometheus/2.33.4" 2024-04-26T16:07:53.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:52 smithi059 ceph-mon[26536]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:53.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:52 smithi059 ceph-mon[21819]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:53.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:55.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:54 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:55.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:54 smithi059 ceph-mon[21819]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:55.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:54 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:55.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:54 smithi059 ceph-mon[26536]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:55.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:54 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:07:55.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:54 smithi064 ceph-mon[22742]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:56.557 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.557 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.557 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:56.557 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:07:56.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:07:56.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:07:56.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:07:56.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:07:56.558 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:07:56.558 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.558 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:56.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:07:56.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:07:56.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:07:56.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:07:56.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:07:56.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:56 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:07:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:56 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:07:57.488 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:57 smithi059 ceph-mon[26536]: Deploying daemon mgr.y on smithi059 2024-04-26T16:07:57.489 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:57 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:57.489 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:57 smithi059 ceph-mon[21819]: Deploying daemon mgr.y on smithi059 2024-04-26T16:07:57.489 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:57 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:57 smithi064 ceph-mon[22742]: Deploying daemon mgr.y on smithi059 2024-04-26T16:07:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:57 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:07:57.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:57 smithi059 systemd[1]: Stopping Ceph mgr.y for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:07:58.244 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:57 smithi059 podman[60157]: 2024-04-26 16:07:57.994734335 +0000 UTC m=+0.154232123 container died 62b620913a03c3654aa6d068692434a6cc9d782ff986351840a534b4d86db799 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, RELEASE=HEAD, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, version=8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, ceph=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=) 2024-04-26T16:07:58.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:07:58 smithi059 ceph-mon[21819]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:58.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:07:58 smithi059 ceph-mon[26536]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:58.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:58 smithi059 podman[60157]: 2024-04-26 16:07:58.397480407 +0000 UTC m=+0.556978213 container cleanup 62b620913a03c3654aa6d068692434a6cc9d782ff986351840a534b4d86db799 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, ceph=True, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-26T16:07:58.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:58 smithi059 bash[60157]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y 2024-04-26T16:07:58.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:07:58 smithi064 ceph-mon[22742]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:07:58.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:07:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:07:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:07:59.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:58 smithi059 podman[60171]: 2024-04-26 16:07:58.7408822 +0000 UTC m=+0.750155965 container remove 62b620913a03c3654aa6d068692434a6cc9d782ff986351840a534b4d86db799 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , name=centos-stream, release=754, GIT_CLEAN=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-26T16:07:59.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:58 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-26T16:07:59.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:58 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-26T16:07:59.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:58 smithi059 systemd[1]: Stopped Ceph mgr.y for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:07:59.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:58 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y.service: Consumed 1min 3.371s CPU time. 2024-04-26T16:07:59.358 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:59 smithi059 systemd[1]: Starting Ceph mgr.y for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:07:59.358 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:59 smithi059 podman[60300]: 2024-04-26 16:07:59.237194059 +0000 UTC m=+0.020861202 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:07:59.622 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:59 smithi059 podman[60300]: 2024-04-26 16:07:59.359164484 +0000 UTC m=+0.142831683 container create c5d6af75064d0a51f8d186230fa316ce3df7c281e67f68549e3bfe6bf08f1615 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:07:59.623 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:59 smithi059 podman[60300]: 2024-04-26 16:07:59.59531301 +0000 UTC m=+0.378980133 container init c5d6af75064d0a51f8d186230fa316ce3df7c281e67f68549e3bfe6bf08f1615 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True) 2024-04-26T16:07:59.623 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:59 smithi059 podman[60300]: 2024-04-26 16:07:59.598054907 +0000 UTC m=+0.381722030 container start c5d6af75064d0a51f8d186230fa316ce3df7c281e67f68549e3bfe6bf08f1615 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-26T16:07:59.955 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:59 smithi059 ceph-mgr[60319]: -- 172.21.15.59:0/3030029929 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5571416525a0 con 0x557141626800 2024-04-26T16:07:59.955 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:59 smithi059 bash[60300]: c5d6af75064d0a51f8d186230fa316ce3df7c281e67f68549e3bfe6bf08f1615 2024-04-26T16:07:59.955 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:59 smithi059 systemd[1]: Started Ceph mgr.y for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:07:59.955 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:07:59.755+0000 7f4d070f8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T16:07:59.955 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:07:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:07:59.954+0000 7f4d070f8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:08:00.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:00.065+0000 7f4d070f8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T16:08:00.735 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:00.415+0000 7f4d070f8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T16:08:00.735 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:00.511+0000 7f4d070f8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T16:08:01.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[21819]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:01.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:08:01.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[26536]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:01.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:00 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:08:01.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:00 smithi064 ceph-mon[22742]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:01.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:00 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:00 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:00 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:00 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:01.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:00 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:08:02.237 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:01.908+0000 7f4d070f8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T16:08:02.238 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:02.113+0000 7f4d070f8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T16:08:02.514 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:02.296+0000 7f4d070f8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T16:08:02.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:02.609+0000 7f4d070f8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T16:08:02.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:02.707+0000 7f4d070f8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T16:08:02.989 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:02 smithi064 ceph-mon[22742]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:02.989 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:08:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:08:02] "GET /metrics HTTP/1.1" 200 37964 "" "Prometheus/2.33.4" 2024-04-26T16:08:03.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:02 smithi059 ceph-mon[21819]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:03.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:02.937+0000 7f4d070f8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T16:08:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:02 smithi059 ceph-mon[26536]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:03.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:03.147+0000 7f4d070f8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:08:03.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:04.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:03.803+0000 7f4d070f8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T16:08:04.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:04.045+0000 7f4d070f8200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-04-26T16:08:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:04.502+0000 7f4d070f8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T16:08:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:04.604+0000 7f4d070f8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T16:08:05.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:04.993+0000 7f4d070f8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T16:08:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[21819]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:08:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:08:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:08:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:05.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:05.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[26536]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:08:05.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:05.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:05.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:08:05.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:08:05.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:04 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:05.181 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:04 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:05.182 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:04 smithi064 ceph-mon[22742]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:08:05.182 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:04 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:05.182 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:04 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:05.182 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:04 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:08:05.182 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:04 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:08:05.182 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:04 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:08:05.730 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:05.371+0000 7f4d070f8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T16:08:05.730 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:05.470+0000 7f4d070f8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T16:08:05.730 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:05.729+0000 7f4d070f8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T16:08:06.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:05.871+0000 7f4d070f8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T16:08:06.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:05.972+0000 7f4d070f8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T16:08:06.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:06.312+0000 7f4d070f8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T16:08:06.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:06.410+0000 7f4d070f8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T16:08:07.049 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:06 smithi059 ceph-mon[26536]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:08:07.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:06 smithi059 ceph-mon[21819]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:08:07.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:06 smithi064 ceph-mon[22742]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:08:07.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:07.049+0000 7f4d070f8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T16:08:07.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:07.284+0000 7f4d070f8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T16:08:07.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:08:07] ENGINE Bus STARTING 2024-04-26T16:08:07.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CherryPy Checker: 2024-04-26T16:08:07.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: The Application mounted at '' has an empty config. 2024-04-26T16:08:07.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:08:07.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:08:07] ENGINE Serving on http://:::9283 2024-04-26T16:08:07.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:08:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:08:07] ENGINE Bus STARTED 2024-04-26T16:08:08.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[21819]: Standby manager daemon y restarted 2024-04-26T16:08:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[21819]: Standby manager daemon y started 2024-04-26T16:08:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T16:08:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:08:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T16:08:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:08:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[21819]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:08:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[26536]: Standby manager daemon y restarted 2024-04-26T16:08:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[26536]: Standby manager daemon y started 2024-04-26T16:08:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T16:08:08.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:08:08.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T16:08:08.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:08:08.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:08 smithi059 ceph-mon[26536]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:08:08.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:08 smithi064 ceph-mon[22742]: Standby manager daemon y restarted 2024-04-26T16:08:08.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:08 smithi064 ceph-mon[22742]: Standby manager daemon y started 2024-04-26T16:08:08.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:08 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T16:08:08.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:08 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:08:08.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:08 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T16:08:08.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:08 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:08:08.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:08 smithi064 ceph-mon[22742]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:08:08.936 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:09.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:09 smithi059 ceph-mon[21819]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-26T16:08:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:09 smithi059 ceph-mon[26536]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-26T16:08:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:09 smithi064 ceph-mon[22742]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-26T16:08:10.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:10 smithi059 ceph-mon[21819]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:08:10.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:10 smithi059 ceph-mon[26536]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:08:10.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:10 smithi064 ceph-mon[22742]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:08:12.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:12 smithi059 ceph-mon[21819]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:08:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:12 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:12 smithi059 ceph-mon[26536]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:08:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:12 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:12.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:12 smithi064 ceph-mon[22742]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:08:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:12 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:13.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:08:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:08:12] "GET /metrics HTTP/1.1" 200 37968 "" "Prometheus/2.33.4" 2024-04-26T16:08:13.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:14 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:14 smithi064 ceph-mon[22742]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:08:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:14 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:14 smithi059 ceph-mon[26536]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:08:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:14 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:14 smithi059 ceph-mon[21819]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:08:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:16 smithi064 ceph-mon[22742]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:16.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:16 smithi059 ceph-mon[21819]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:16 smithi059 ceph-mon[26536]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:18.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:18 smithi059 ceph-mon[21819]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:18 smithi059 ceph-mon[26536]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:18.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:18 smithi064 ceph-mon[22742]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:18.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:20 smithi064 ceph-mon[22742]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:20.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:20 smithi059 ceph-mon[21819]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:20 smithi059 ceph-mon[26536]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:22.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:22 smithi064 ceph-mon[22742]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:22.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:22 smithi059 ceph-mon[21819]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:22.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:22 smithi059 ceph-mon[26536]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:23.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:08:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:08:22] "GET /metrics HTTP/1.1" 200 37968 "" "Prometheus/2.33.4" 2024-04-26T16:08:23.936 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:24.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:24 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:24 smithi064 ceph-mon[22742]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:24.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:24 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:24.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:24 smithi059 ceph-mon[21819]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:24 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:24 smithi059 ceph-mon[26536]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:26.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:26 smithi064 ceph-mon[22742]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:26.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:26 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:26.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:26 smithi059 ceph-mon[26536]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:26 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:26 smithi059 ceph-mon[21819]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:26 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:28.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:28 smithi059 ceph-mon[21819]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:28.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:28 smithi059 ceph-mon[26536]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:28.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:28 smithi064 ceph-mon[22742]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:28.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:30 smithi064 ceph-mon[22742]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:30.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:30 smithi059 ceph-mon[26536]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:30 smithi059 ceph-mon[21819]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:32.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:32 smithi064 ceph-mon[22742]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:32.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:32 smithi059 ceph-mon[26536]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:32.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:32 smithi059 ceph-mon[21819]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:33.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:08:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:08:32] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-04-26T16:08:33.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:33 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:34.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:34 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:34 smithi064 ceph-mon[22742]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:34.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:34 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:34 smithi059 ceph-mon[21819]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:34 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:34 smithi059 ceph-mon[26536]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:36.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:36 smithi064 ceph-mon[22742]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:36.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:36 smithi059 ceph-mon[26536]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:36.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:36 smithi059 ceph-mon[21819]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:38.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:38 smithi059 ceph-mon[21819]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:38.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:38 smithi059 ceph-mon[26536]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:38.657 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:38 smithi064 ceph-mon[22742]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:38.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:38.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:40 smithi064 ceph-mon[22742]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:40.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:40 smithi059 ceph-mon[21819]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:40 smithi059 ceph-mon[26536]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:41.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:41 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:41.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:41 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:41 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:42.700 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:42 smithi064 ceph-mon[22742]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:42.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:42 smithi059 ceph-mon[26536]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:42.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:42 smithi059 ceph-mon[21819]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:43.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:08:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:08:42] "GET /metrics HTTP/1.1" 200 37966 "" "Prometheus/2.33.4" 2024-04-26T16:08:43.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:44 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:44 smithi059 ceph-mon[26536]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:44 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:44 smithi059 ceph-mon[21819]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:44.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:44 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:44 smithi064 ceph-mon[22742]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:46.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:46 smithi059 ceph-mon[21819]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:46.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:46 smithi059 ceph-mon[26536]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:46.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:46 smithi064 ceph-mon[22742]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:48 smithi059 ceph-mon[21819]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:48.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:48 smithi059 ceph-mon[26536]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:48.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:48 smithi064 ceph-mon[22742]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:48.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:50.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:50 smithi064 ceph-mon[22742]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:50.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:50 smithi059 ceph-mon[21819]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:50.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:50 smithi059 ceph-mon[26536]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:52.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:52 smithi064 ceph-mon[22742]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:52.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:52 smithi059 ceph-mon[26536]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:52 smithi059 ceph-mon[21819]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:53.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:08:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:08:52] "GET /metrics HTTP/1.1" 200 37966 "" "Prometheus/2.33.4" 2024-04-26T16:08:53.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:08:54.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:54 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:54.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:54 smithi059 ceph-mon[21819]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:54 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:54 smithi059 ceph-mon[26536]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:54.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:54 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:08:54.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:54 smithi064 ceph-mon[22742]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:56.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:56 smithi059 ceph-mon[21819]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:56.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:56 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:56 smithi059 ceph-mon[26536]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:56 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:56.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:56 smithi064 ceph-mon[22742]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:08:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:56 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:08:58.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:08:58 smithi059 ceph-mon[21819]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:58.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:08:58 smithi059 ceph-mon[26536]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:58.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:08:58 smithi064 ceph-mon[22742]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:08:58.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:08:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:08:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:00 smithi064 ceph-mon[22742]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:00.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:00 smithi059 ceph-mon[26536]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:00.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:00 smithi059 ceph-mon[21819]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:02.700 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:02 smithi064 ceph-mon[22742]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:02.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:02 smithi059 ceph-mon[21819]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:02.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:02 smithi059 ceph-mon[26536]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:03.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:09:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:09:02] "GET /metrics HTTP/1.1" 200 37963 "" "Prometheus/2.33.4" 2024-04-26T16:09:03.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:03.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:04.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:04 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:04.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:04 smithi059 ceph-mon[26536]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:04 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:09:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:04 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:04 smithi059 ceph-mon[21819]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:04 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:09:04.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:04 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:04.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:04 smithi064 ceph-mon[22742]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:04 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:09:05.447 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:05 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:09:05.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:05 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:09:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:05 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:09:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:05 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:09:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:05 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:09:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:05 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:09:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:05 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:09:05.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:05 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:09:05.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:05 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:09:06.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:06 smithi059 ceph-mon[26536]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:06 smithi059 ceph-mon[21819]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:06.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:06 smithi064 ceph-mon[22742]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:08 smithi059 ceph-mon[26536]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:08.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:08 smithi059 ceph-mon[21819]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:08.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:08 smithi064 ceph-mon[22742]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:08.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:10.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:10 smithi059 ceph-mon[26536]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:10 smithi059 ceph-mon[21819]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:10.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:10 smithi064 ceph-mon[22742]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:11.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:11 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:11.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:11 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:11 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:12.700 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:12 smithi064 ceph-mon[22742]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:12.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:12 smithi059 ceph-mon[26536]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:12 smithi059 ceph-mon[21819]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:13.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:09:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:09:12] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-26T16:09:13.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:14.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:14 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:14.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:14 smithi059 ceph-mon[21819]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:14.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:14 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:14.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:14 smithi059 ceph-mon[26536]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:14.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:14 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:14 smithi064 ceph-mon[22742]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:16.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:16 smithi059 ceph-mon[26536]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:16 smithi059 ceph-mon[21819]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:16.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:16 smithi064 ceph-mon[22742]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:18 smithi059 ceph-mon[26536]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:18.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:18 smithi059 ceph-mon[21819]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:18.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:18 smithi064 ceph-mon[22742]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:18.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:20.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:20 smithi059 ceph-mon[26536]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:20 smithi059 ceph-mon[21819]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:20.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:20 smithi064 ceph-mon[22742]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:22.700 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:22 smithi064 ceph-mon[22742]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:22.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:22 smithi059 ceph-mon[26536]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:22.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:22 smithi059 ceph-mon[21819]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:23.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:09:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:09:22] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-26T16:09:23.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:24.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:24 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:24.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:24 smithi059 ceph-mon[21819]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:24 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:24 smithi059 ceph-mon[26536]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:24 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:24 smithi064 ceph-mon[22742]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:26.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:26 smithi059 ceph-mon[21819]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:26 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:26 smithi059 ceph-mon[26536]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:26 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:26.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:26 smithi064 ceph-mon[22742]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:26 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:28.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:28 smithi059 ceph-mon[21819]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:28.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:28 smithi059 ceph-mon[26536]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:28.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:28 smithi064 ceph-mon[22742]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:28.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:30 smithi059 ceph-mon[21819]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:30 smithi059 ceph-mon[26536]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:30.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:30 smithi064 ceph-mon[22742]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:32.738 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:32 smithi064 ceph-mon[22742]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:32.739 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:09:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:09:32] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-26T16:09:32.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:32 smithi059 ceph-mon[21819]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:32.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:32 smithi059 ceph-mon[26536]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:33.936 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:33 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:34 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:34 smithi059 ceph-mon[21819]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:34 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:34 smithi059 ceph-mon[26536]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:34.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:34 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:34.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:34 smithi064 ceph-mon[22742]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:36.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:36 smithi059 ceph-mon[21819]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:36.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:36 smithi059 ceph-mon[26536]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:36 smithi064 ceph-mon[22742]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:38.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:38 smithi059 ceph-mon[21819]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:38.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:38 smithi059 ceph-mon[26536]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:38.659 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:38 smithi064 ceph-mon[22742]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:38.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:38 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:40.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:40 smithi059 ceph-mon[21819]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:40 smithi059 ceph-mon[26536]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:40.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:40 smithi064 ceph-mon[22742]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:41.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:41 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:41 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:41.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:41 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:42.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:42 smithi059 ceph-mon[21819]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:42.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:42 smithi059 ceph-mon[26536]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:42 smithi064 ceph-mon[22742]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:42.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:09:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:09:42] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-26T16:09:43.686 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:44.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:44 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:44 smithi059 ceph-mon[21819]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:44 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:44 smithi059 ceph-mon[26536]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:44 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:44 smithi064 ceph-mon[22742]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:46.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:46 smithi059 ceph-mon[21819]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:46.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:46 smithi059 ceph-mon[26536]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:46.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:46 smithi064 ceph-mon[22742]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:48.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:48 smithi059 ceph-mon[21819]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:48.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:48 smithi059 ceph-mon[26536]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:48.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:48 smithi064 ceph-mon[22742]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:48.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:50.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:50 smithi059 ceph-mon[21819]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:50.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:50 smithi059 ceph-mon[26536]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:50.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:50 smithi064 ceph-mon[22742]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:52.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:52 smithi059 ceph-mon[21819]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:52.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:52 smithi059 ceph-mon[26536]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:52.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:52 smithi064 ceph-mon[22742]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:52.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:09:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:09:52] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-26T16:09:53.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi059\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi059\", job=\"node\", machine=\"x86_64\", nodename=\"smithi059\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:09:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:54 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:54 smithi059 ceph-mon[26536]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:54.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:54 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:54.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:54 smithi059 ceph-mon[21819]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:54.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:54 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:09:54.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:54 smithi064 ceph-mon[22742]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:56.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:56 smithi059 ceph-mon[26536]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:56 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:56.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:56 smithi059 ceph-mon[21819]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:56.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:56 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:56 smithi064 ceph-mon[22742]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:09:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:56 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:09:58.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:09:58 smithi059 ceph-mon[26536]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:58.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:09:58 smithi059 ceph-mon[21819]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:58.658 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:09:58 smithi064 ceph-mon[22742]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:09:58.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:09:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:09:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"172.21.15.64:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"b6607c98-03e4-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.59\", device_class=\"ssd\", hostname=\"smithi059\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.59\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T16:10:00.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:00 smithi059 ceph-mon[21819]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:00.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:00 smithi059 ceph-mon[21819]: overall HEALTH_OK 2024-04-26T16:10:00.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:00 smithi059 ceph-mon[26536]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:00.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:00 smithi059 ceph-mon[26536]: overall HEALTH_OK 2024-04-26T16:10:00.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:00 smithi064 ceph-mon[22742]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:00.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:00 smithi064 ceph-mon[22742]: overall HEALTH_OK 2024-04-26T16:10:02.739 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:02 smithi064 ceph-mon[22742]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:02.739 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:10:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:10:02] "GET /metrics HTTP/1.1" 200 37964 "" "Prometheus/2.33.4" 2024-04-26T16:10:02.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:02 smithi059 ceph-mon[21819]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:02.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:02 smithi059 ceph-mon[26536]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:04.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:04 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:04 smithi059 ceph-mon[21819]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:04 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:04 smithi059 ceph-mon[26536]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:04 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:04 smithi064 ceph-mon[22742]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:05.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:05 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:10:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:05 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:10:05.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:05 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:10:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:06 smithi059 ceph-mon[21819]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:06 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:10:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:06 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:10:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:06 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:10:06.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:06 smithi059 ceph-mon[26536]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:06.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:06 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:10:06.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:06 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:10:06.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:06 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:10:06.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:06 smithi064 ceph-mon[22742]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:06.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:06 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:10:06.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:06 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:10:06.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:06 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:10:08.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:08 smithi059 ceph-mon[21819]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:08 smithi059 ceph-mon[26536]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:08 smithi064 ceph-mon[22742]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:10.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:10 smithi059 ceph-mon[21819]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:10.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:10 smithi059 ceph-mon[26536]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:10.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:10 smithi064 ceph-mon[22742]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:11.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:11 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:11.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:11 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:11.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:11 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:12.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:12 smithi059 ceph-mon[21819]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:12.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:12 smithi059 ceph-mon[26536]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:12.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:12 smithi064 ceph-mon[22742]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:12.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:10:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:10:12] "GET /metrics HTTP/1.1" 200 37967 "" "Prometheus/2.33.4" 2024-04-26T16:10:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:14 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:14 smithi059 ceph-mon[21819]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:14 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:14 smithi059 ceph-mon[26536]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:14 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:14 smithi064 ceph-mon[22742]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:16.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:16 smithi059 ceph-mon[21819]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:16 smithi059 ceph-mon[26536]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:16.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:16 smithi064 ceph-mon[22742]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:18.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:18 smithi059 ceph-mon[21819]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:18 smithi059 ceph-mon[26536]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:18 smithi064 ceph-mon[22742]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:20.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:20 smithi059 ceph-mon[21819]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:20 smithi059 ceph-mon[26536]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:20.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:20 smithi064 ceph-mon[22742]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:22.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:22 smithi059 ceph-mon[21819]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:22.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:22 smithi059 ceph-mon[26536]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:22.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:22 smithi064 ceph-mon[22742]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:22.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:10:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:10:22] "GET /metrics HTTP/1.1" 200 37967 "" "Prometheus/2.33.4" 2024-04-26T16:10:24.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:24 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:24.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:24 smithi059 ceph-mon[21819]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:24 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:24 smithi059 ceph-mon[26536]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:24.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:24 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:24 smithi064 ceph-mon[22742]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:26.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:26 smithi059 ceph-mon[21819]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:26 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:26 smithi059 ceph-mon[26536]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:26 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:26 smithi064 ceph-mon[22742]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:26 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:28.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:28 smithi059 ceph-mon[21819]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:28.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:28 smithi059 ceph-mon[26536]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:28.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:28 smithi064 ceph-mon[22742]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:30.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:30 smithi059 ceph-mon[26536]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:30 smithi059 ceph-mon[21819]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:30.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:30 smithi064 ceph-mon[22742]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:32.739 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:32 smithi064 ceph-mon[22742]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:32.740 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:10:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:10:32] "GET /metrics HTTP/1.1" 200 37964 "" "Prometheus/2.33.4" 2024-04-26T16:10:32.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:32 smithi059 ceph-mon[26536]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:32.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:32 smithi059 ceph-mon[21819]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:34.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:34 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:34 smithi059 ceph-mon[21819]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:34 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:34 smithi059 ceph-mon[26536]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:34.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:34 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:34.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:34 smithi064 ceph-mon[22742]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:36.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:36 smithi059 ceph-mon[21819]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:36.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:36 smithi059 ceph-mon[26536]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:36 smithi064 ceph-mon[22742]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:38.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:38 smithi059 ceph-mon[21819]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:38.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:38 smithi059 ceph-mon[26536]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:38 smithi064 ceph-mon[22742]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:40.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:40 smithi059 ceph-mon[21819]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:40 smithi059 ceph-mon[26536]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:40.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:40 smithi064 ceph-mon[22742]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:41.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:41 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:41 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:41 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:42.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:42 smithi059 ceph-mon[26536]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:42.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:42 smithi059 ceph-mon[21819]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:42.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:42 smithi064 ceph-mon[22742]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:42.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:10:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:10:42] "GET /metrics HTTP/1.1" 200 37964 "" "Prometheus/2.33.4" 2024-04-26T16:10:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:44 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:44 smithi059 ceph-mon[21819]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:44 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:44 smithi059 ceph-mon[26536]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:44 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:44 smithi064 ceph-mon[22742]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:46.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:46 smithi059 ceph-mon[21819]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:46.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:46 smithi059 ceph-mon[26536]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:46.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:46 smithi064 ceph-mon[22742]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:48.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:48 smithi059 ceph-mon[21819]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:48.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:48 smithi059 ceph-mon[26536]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:48.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:48 smithi064 ceph-mon[22742]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:50.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:50 smithi059 ceph-mon[21819]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:50.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:50 smithi059 ceph-mon[26536]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:50.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:50 smithi064 ceph-mon[22742]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:52.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:52 smithi059 ceph-mon[26536]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:52 smithi059 ceph-mon[21819]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:52.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:52 smithi064 ceph-mon[22742]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:52.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:10:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:10:52] "GET /metrics HTTP/1.1" 200 37964 "" "Prometheus/2.33.4" 2024-04-26T16:10:54.265 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:10:54.576 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:54 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:54.577 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:54 smithi059 ceph-mon[21819]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:54.577 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:54 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:54.577 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:54 smithi059 ceph-mon[26536]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:54.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:54 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:10:54.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:54 smithi064 ceph-mon[22742]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (5m) 2m ago 13m 25.5M - 0.23.0 ba2b418f427c ca6d4a23fcc2 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (5m) 3m ago 13m 50.3M - 8.3.5 dad864ee21e9 0d16158d9791 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 running (5m) 2m ago 12m 53.0M - 3.5 e1d6a67b021e 65a32364fe46 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283 running (8m) 3m ago 18m 523M - 19.0.0-3305-ga430f9ed 031ba2747cfd 03edb3b65e45 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (2m) 2m ago 19m 66.0M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (19m) 2m ago 19m 57.6M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (18m) 3m ago 18m 50.0M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (18m) 2m ago 18m 52.1M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (6m) 2m ago 14m 18.1M - 1.3.1 1dbe0e931976 9b14cfbf9c80 2024-04-26T16:10:56.390 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (5m) 3m ago 14m 18.2M - 1.3.1 1dbe0e931976 fb96f1dc25f3 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (17m) 2m ago 17m 51.7M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (17m) 2m ago 17m 52.5M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (17m) 2m ago 17m 50.6M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (16m) 2m ago 16m 50.9M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (16m) 3m ago 16m 51.7M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (15m) 3m ago 15m 51.9M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (15m) 3m ago 15m 48.5M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (15m) 3m ago 15m 51.1M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (5m) 3m ago 14m 71.9M - 2.33.4 514e6a882f6e 27cf942569e5 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (12m) 2m ago 12m 71.6M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:10:56.391 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (12m) 3m ago 12m 71.7M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:10:56.820 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:56 smithi059 ceph-mon[21819]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:56.820 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:56 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:56.820 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:56 smithi059 ceph-mon[26536]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:56.820 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:56 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:56.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:56 smithi064 ceph-mon[22742]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:10:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:56 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:10:57.081 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions' 2024-04-26T16:10:57.727 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:57 smithi059 ceph-mon[21819]: from='client.25114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:10:57.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:57 smithi059 ceph-mon[26536]: from='client.25114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:10:57.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:57 smithi064 ceph-mon[22742]: from='client.25114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:10:58.650 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:58 smithi059 ceph-mon[26536]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:58.651 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:58 smithi059 ceph-mon[21819]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:58.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:58 smithi064 ceph-mon[22742]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:10:59.173 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:10:59.173 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:10:59.173 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:10:59.173 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:10:59.173 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:10:59.173 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:10:59.174 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:10:59.763 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:10:59 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1909293869' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:10:59.764 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:10:59 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1909293869' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:10:59.788 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:10:59.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:10:59 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1909293869' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:11:00.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:00 smithi059 ceph-mon[26536]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:00.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:00 smithi059 ceph-mon[21819]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:00.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:00 smithi064 ceph-mon[22742]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:01.694 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_OK 2024-04-26T16:11:02.302 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph -s' 2024-04-26T16:11:02.789 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:02 smithi059 ceph-mon[21819]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:02.789 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:02 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/510469344' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:11:02.790 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:02 smithi059 ceph-mon[26536]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:02.790 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:02 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/510469344' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:11:02.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:02 smithi064 ceph-mon[22742]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:02.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:02 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/510469344' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:11:02.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:11:02] "GET /metrics HTTP/1.1" 200 37964 "" "Prometheus/2.33.4" 2024-04-26T16:11:04.239 INFO:teuthology.orchestra.run.smithi059.stdout: cluster: 2024-04-26T16:11:04.239 INFO:teuthology.orchestra.run.smithi059.stdout: id: b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:11:04.239 INFO:teuthology.orchestra.run.smithi059.stdout: health: HEALTH_OK 2024-04-26T16:11:04.239 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:11:04.239 INFO:teuthology.orchestra.run.smithi059.stdout: services: 2024-04-26T16:11:04.239 INFO:teuthology.orchestra.run.smithi059.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-04-26T16:11:04.239 INFO:teuthology.orchestra.run.smithi059.stdout: mgr: x(active, since 6m), standbys: y 2024-04-26T16:11:04.239 INFO:teuthology.orchestra.run.smithi059.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: data: 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: pools: 6 pools, 161 pgs 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: objects: 209 objects, 585 KiB 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: pgs: 161 active+clean 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: io: 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-26T16:11:04.240 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:11:04.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:04 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:04 smithi059 ceph-mon[26536]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:11:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:04 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/544379844' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:11:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:04 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:04 smithi059 ceph-mon[21819]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:11:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:04 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/544379844' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:11:04.859 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph mgr fail' 2024-04-26T16:11:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:04 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:04 smithi064 ceph-mon[22742]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:11:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:04 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/544379844' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:11:06.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:06 smithi059 ceph-mon[21819]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:06 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:11:06.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:06 smithi059 ceph-mon[26536]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:06.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:06 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:11:06.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:06 smithi064 ceph-mon[22742]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:06.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:06 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:11:07.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:07.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-mon[22742]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:11:07.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-mon[22742]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:11:07.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3313635448' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:11:07.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:11:07.771 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-mon[22742]: osdmap e104: 8 total, 8 up, 8 in 2024-04-26T16:11:07.786 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:07.786 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[26536]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:11:07.786 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[26536]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:11:07.786 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3313635448' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:11:07.786 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:11:07.787 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[26536]: osdmap e104: 8 total, 8 up, 8 in 2024-04-26T16:11:07.787 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:07.787 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[21819]: from='mgr.34820 172.21.15.64:0/3688267835' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:11:07.787 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[21819]: from='mgr.34820 ' entity='mgr.x' 2024-04-26T16:11:07.787 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3313635448' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:11:07.787 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:11:07.787 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-mon[21819]: osdmap e104: 8 total, 8 up, 8 in 2024-04-26T16:11:08.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:07] ENGINE Bus STOPPING 2024-04-26T16:11:08.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:11:08.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:07] ENGINE Bus STOPPED 2024-04-26T16:11:08.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:07] ENGINE Bus STARTING 2024-04-26T16:11:08.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:07] ENGINE Serving on http://:::9283 2024-04-26T16:11:08.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:07 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:07] ENGINE Bus STARTED 2024-04-26T16:11:08.150 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:07.771+0000 7f840d51b700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T16:11:08.150 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ignoring --setuser ceph since I am not root 2024-04-26T16:11:08.150 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ignoring --setgroup ceph since I am not root 2024-04-26T16:11:08.151 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-mgr[42967]: -- 172.21.15.64:0/1269416728 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5605bb52a5a0 con 0x5605bb4fe800 2024-04-26T16:11:08.151 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:07.957+0000 7f15c9df3200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T16:11:08.151 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:08.150+0000 7f15c9df3200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:11:08.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:08.262+0000 7f15c9df3200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T16:11:08.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: Standby manager daemon x started 2024-04-26T16:11:08.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-26T16:11:08.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: mgrmap e28: y(active, starting, since 0.866069s), standbys: x 2024-04-26T16:11:08.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:11:08.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: Manager daemon y is now available 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: Standby manager daemon x started 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: mgrmap e28: y(active, starting, since 0.866069s), standbys: x 2024-04-26T16:11:08.527 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:11:08.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:11:08.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: Manager daemon y is now available 2024-04-26T16:11:08.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:11:08.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:08.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:11:08.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:11:08.550 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'sleep 180' 2024-04-26T16:11:08.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:08.596+0000 7f15c9df3200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T16:11:08.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:08.696+0000 7f15c9df3200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T16:11:08.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:08.631Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-04-26T16:11:08.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:08.632Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-04-26T16:11:08.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:08.633Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-04-26T16:11:08.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:08.634Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-04-26T16:11:08.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:08.634Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-04-26T16:11:08.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: Standby manager daemon x started 2024-04-26T16:11:08.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-26T16:11:08.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: mgrmap e28: y(active, starting, since 0.866069s), standbys: x 2024-04-26T16:11:08.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:11:08.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:11:08.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:11:08.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:11:08.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:11:08.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:11:08.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: Manager daemon y is now available 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:11:08.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:11:09.808 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:09 smithi064 ceph-mon[22742]: [26/Apr/2024:16:11:08] ENGINE Bus STARTING 2024-04-26T16:11:09.809 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:09 smithi064 ceph-mon[22742]: mgrmap e29: y(active, since 1.86901s), standbys: x 2024-04-26T16:11:09.823 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:09 smithi059 ceph-mon[26536]: [26/Apr/2024:16:11:08] ENGINE Bus STARTING 2024-04-26T16:11:09.823 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:09 smithi059 ceph-mon[26536]: mgrmap e29: y(active, since 1.86901s), standbys: x 2024-04-26T16:11:09.823 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:09 smithi059 ceph-mon[21819]: [26/Apr/2024:16:11:08] ENGINE Bus STARTING 2024-04-26T16:11:09.824 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:09 smithi059 ceph-mon[21819]: mgrmap e29: y(active, since 1.86901s), standbys: x 2024-04-26T16:11:10.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:10.010+0000 7f15c9df3200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T16:11:10.505 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:10.189+0000 7f15c9df3200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T16:11:10.506 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:10.403+0000 7f15c9df3200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T16:11:10.782 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:10 smithi064 ceph-mon[22742]: [26/Apr/2024:16:11:08] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T16:11:10.782 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:10 smithi064 ceph-mon[22742]: [26/Apr/2024:16:11:08] ENGINE Serving on http://172.21.15.59:8765 2024-04-26T16:11:10.782 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:10 smithi064 ceph-mon[22742]: [26/Apr/2024:16:11:08] ENGINE Bus STARTED 2024-04-26T16:11:10.782 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:10 smithi064 ceph-mon[22742]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:10.782 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:10 smithi064 ceph-mon[22742]: mgrmap e30: y(active, since 2s), standbys: x 2024-04-26T16:11:10.783 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:10.755+0000 7f15c9df3200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T16:11:10.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:10 smithi059 ceph-mon[26536]: [26/Apr/2024:16:11:08] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T16:11:10.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:10 smithi059 ceph-mon[26536]: [26/Apr/2024:16:11:08] ENGINE Serving on http://172.21.15.59:8765 2024-04-26T16:11:10.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:10 smithi059 ceph-mon[26536]: [26/Apr/2024:16:11:08] ENGINE Bus STARTED 2024-04-26T16:11:10.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:10 smithi059 ceph-mon[26536]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:10.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:10 smithi059 ceph-mon[26536]: mgrmap e30: y(active, since 2s), standbys: x 2024-04-26T16:11:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:10 smithi059 ceph-mon[21819]: [26/Apr/2024:16:11:08] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T16:11:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:10 smithi059 ceph-mon[21819]: [26/Apr/2024:16:11:08] ENGINE Serving on http://172.21.15.59:8765 2024-04-26T16:11:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:10 smithi059 ceph-mon[21819]: [26/Apr/2024:16:11:08] ENGINE Bus STARTED 2024-04-26T16:11:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:10 smithi059 ceph-mon[21819]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:10 smithi059 ceph-mon[21819]: mgrmap e30: y(active, since 2s), standbys: x 2024-04-26T16:11:11.072 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:10.855+0000 7f15c9df3200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T16:11:11.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:11.072+0000 7f15c9df3200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T16:11:11.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:11.309+0000 7f15c9df3200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:11:11.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:11 smithi059 ceph-mon[21819]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:11.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:11 smithi059 ceph-mon[26536]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:11.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:11 smithi064 ceph-mon[22742]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:12.226 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:11.975+0000 7f15c9df3200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T16:11:12.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:12.225+0000 7f15c9df3200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-04-26T16:11:12.950 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:12 smithi059 ceph-mon[21819]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:12.950 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:12 smithi059 ceph-mon[21819]: mgrmap e31: y(active, since 4s), standbys: x 2024-04-26T16:11:12.951 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:12 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:12.951 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:12 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:12.951 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:12 smithi059 ceph-mon[26536]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:12.951 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:12 smithi059 ceph-mon[26536]: mgrmap e31: y(active, since 4s), standbys: x 2024-04-26T16:11:12.951 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:12 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:12.951 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:12 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:13.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:12.697+0000 7f15c9df3200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T16:11:13.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:12.818+0000 7f15c9df3200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T16:11:13.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:12 smithi064 ceph-mon[22742]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:13.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:12 smithi064 ceph-mon[22742]: mgrmap e31: y(active, since 4s), standbys: x 2024-04-26T16:11:13.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:12 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:13.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:12 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:13.621 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:13.221+0000 7f15c9df3200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T16:11:13.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:13 smithi064 ceph-mon[22742]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:13.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:13.620+0000 7f15c9df3200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T16:11:13.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:13.722+0000 7f15c9df3200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T16:11:14.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:13 smithi059 ceph-mon[26536]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:14.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:13 smithi059 ceph-mon[21819]: from='client.34952 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:14.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:13.991+0000 7f15c9df3200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T16:11:14.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:14.133+0000 7f15c9df3200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T16:11:14.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:14.235+0000 7f15c9df3200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T16:11:14.837 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:14.577+0000 7f15c9df3200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T16:11:14.837 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:14.675+0000 7f15c9df3200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T16:11:15.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:14 smithi064 ceph-mon[22742]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:15.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:14 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:14 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:14 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:14 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:14 smithi059 ceph-mon[26536]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:15.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:14 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:14 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:14 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:14 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:14 smithi059 ceph-mon[21819]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:15.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:14 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:14 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:14 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:14 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:15.666 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:15.319+0000 7f15c9df3200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T16:11:15.666 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:15.558+0000 7f15c9df3200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T16:11:15.666 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:11:15] ENGINE Bus STARTING 2024-04-26T16:11:15.667 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: CherryPy Checker: 2024-04-26T16:11:15.667 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: The Application mounted at '' has an empty config. 2024-04-26T16:11:15.667 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:11:15.667 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:11:15] ENGINE Serving on http://:::9283 2024-04-26T16:11:15.667 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:11:15] ENGINE Bus STARTED 2024-04-26T16:11:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:11:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:11:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:11:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:11:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:11:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: Standby manager daemon x restarted 2024-04-26T16:11:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: Standby manager daemon x started 2024-04-26T16:11:16.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:11:16.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:11:16.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:11:16.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:11:16.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:11:16.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:16 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: Standby manager daemon x restarted 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: Standby manager daemon x started 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:11:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:11:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:11:16.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: Standby manager daemon x restarted 2024-04-26T16:11:16.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: Standby manager daemon x started 2024-04-26T16:11:16.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:11:16.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:11:16.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:11:16.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:11:16.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:11:16.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:16 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:11:17.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:11:17.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:11:17.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:11:17.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: mgrmap e32: y(active, since 9s), standbys: x 2024-04-26T16:11:17.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:11:17.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:11:17.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:17.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:11:17.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:11:17.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:11:17.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:11:17.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: mgrmap e32: y(active, since 9s), standbys: x 2024-04-26T16:11:17.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:11:17.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:11:17.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:17.336 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:17 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:11:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:11:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:11:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:11:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: mgrmap e32: y(active, since 9s), standbys: x 2024-04-26T16:11:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:17.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:11:17.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:11:17.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:17.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:17 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:11:19.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-mon[26536]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:11:19.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-mon[21819]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:11:19.301+0000 7f4c8bc6e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:11:19.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:11:19.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:19.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:19.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:19.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:19.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:19.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:19.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:19.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:19 smithi064 ceph-mon[22742]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:11:20.417 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:20 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:20.417 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:20.417 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:20.418 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:11:20.419 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:20.420 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.421 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.421 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:11:20.421 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:20.421 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:20.421 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:20.421 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:11:20.421 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:20.421 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:20.421 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:20.421 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:20.422 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:20.422 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:20.422 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:20.422 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.422 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:20 smithi064 ceph-mon[22742]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:11:20.422 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:20 smithi064 ceph-mon[22742]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:11:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:20 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:11:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:20 smithi059 ceph-mon[21819]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:20 smithi059 ceph-mon[21819]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:11:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:20 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:11:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:20 smithi059 ceph-mon[26536]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:11:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:20 smithi059 ceph-mon[26536]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:11:21.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:21 smithi064 ceph-mon[22742]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:11:21.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:21 smithi059 ceph-mon[21819]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:11:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:21 smithi059 ceph-mon[26536]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:11:22.877 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:11:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-26T16:11:22.878 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 systemd[1]: Stopping Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:11:22.878 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 bash[51617]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-26T16:11:23.153 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.878Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T16:11:23.153 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.878Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-26T16:11:23.153 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.878Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-26T16:11:23.153 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.878Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-26T16:11:23.153 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.878Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-26T16:11:23.153 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.878Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-26T16:11:23.153 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.878Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T16:11:23.153 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.878Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-26T16:11:23.153 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.878Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T16:11:23.153 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.879Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T16:11:23.154 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.879Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-26T16:11:23.154 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[46124]: ts=2024-04-26T16:11:22.879Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-26T16:11:23.154 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:22 smithi064 podman[51630]: 2024-04-26 16:11:22.888651551 +0000 UTC m=+0.109648246 container died 27cf942569e58a068ef5ec388abef93d7f29864872f361dd97acc389c94fd02a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:11:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-mon[22742]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:11:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:23.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 podman[51630]: 2024-04-26 16:11:23.243940462 +0000 UTC m=+0.464937146 container cleanup 27cf942569e58a068ef5ec388abef93d7f29864872f361dd97acc389c94fd02a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:11:23.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 bash[51630]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T16:11:23.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 bash[51656]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-26T16:11:23.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:23 smithi059 ceph-mon[26536]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:11:23.646 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:23 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:23.647 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:23 smithi059 ceph-mon[21819]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:11:23.647 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:23 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:23.834 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 podman[51644]: 2024-04-26 16:11:23.52054284 +0000 UTC m=+0.630928781 container remove 27cf942569e58a068ef5ec388abef93d7f29864872f361dd97acc389c94fd02a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:11:23.834 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T16:11:23.834 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 systemd[1]: Stopped Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:11:23.834 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 2.015s CPU time. 2024-04-26T16:11:23.834 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 systemd[1]: Starting Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:11:23.834 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 podman[51728]: 2024-04-26 16:11:23.73809429 +0000 UTC m=+0.106712479 container create 25fb46b85a951e02079fcb4c41c421f1555909da2b939df37ae2ea5c2bc0e4fb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:11:23.834 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 podman[51728]: 2024-04-26 16:11:23.65076729 +0000 UTC m=+0.019385462 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-26T16:11:24.187 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 podman[51728]: 2024-04-26 16:11:23.949628758 +0000 UTC m=+0.318246908 container init 25fb46b85a951e02079fcb4c41c421f1555909da2b939df37ae2ea5c2bc0e4fb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:11:24.187 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 podman[51728]: 2024-04-26 16:11:23.952435772 +0000 UTC m=+0.321053926 container start 25fb46b85a951e02079fcb4c41c421f1555909da2b939df37ae2ea5c2bc0e4fb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.978Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.978Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.978Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.978Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi064 (none))" 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.978Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.978Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.979Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.980Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.981Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.983Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.984Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.380787ms 2024-04-26T16:11:24.188 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:23.984Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T16:11:24.189 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 bash[51728]: 25fb46b85a951e02079fcb4c41c421f1555909da2b939df37ae2ea5c2bc0e4fb 2024-04-26T16:11:24.189 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 systemd[1]: Started Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:11:24.471 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:24.199Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-26T16:11:24.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:24] ENGINE Bus STOPPING 2024-04-26T16:11:24.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:11:24.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:24] ENGINE Bus STOPPED 2024-04-26T16:11:24.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:24] ENGINE Bus STARTING 2024-04-26T16:11:24.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:24] ENGINE Serving on http://:::9283 2024-04-26T16:11:24.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:11:24] ENGINE Bus STARTED 2024-04-26T16:11:24.920 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:24.472Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-26T16:11:24.920 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:24.680Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-26T16:11:24.920 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:24.680Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-26T16:11:24.920 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:24.680Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=52.335µs wal_replay_duration=696.078927ms total_replay_duration=697.536611ms 2024-04-26T16:11:24.920 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:24.685Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T16:11:24.920 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:24.685Z caller=main.go:947 level=info msg="TSDB started" 2024-04-26T16:11:24.920 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:24.685Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T16:11:24.920 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:24.707Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.559305ms db_storage=782ns remote_storage=1.292µs web_handler=308ns query_engine=720ns scrape=242.321µs scrape_sd=95.005µs notify=13.786µs notify_sd=16.317µs rules=20.776228ms 2024-04-26T16:11:24.920 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:11:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:11:24.707Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-26T16:11:25.182 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:25 smithi064 ceph-mon[22742]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:11:25.182 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:25 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:25.182 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:25 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:25.183 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:25 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:11:25.183 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:25 smithi064 ceph-mon[22742]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:11:25.183 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:25 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:11:25.183 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:25 smithi064 ceph-mon[22742]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:11:25.183 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:25 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:25.183 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:25 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:11:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[26536]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:11:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:11:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[26536]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:11:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:11:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[26536]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:11:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:11:25.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[21819]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:11:25.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:25.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:25.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:11:25.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[21819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:11:25.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:11:25.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[21819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:11:25.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:25.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:25 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:11:27.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:27 smithi059 ceph-mon[26536]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:11:27.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:27 smithi059 ceph-mon[21819]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:11:27.372 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:27 smithi064 ceph-mon[22742]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T16:11:29.443 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[21819]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:11:29.443 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:29.443 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[26536]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:29.444 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:29.445 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:11:29.445 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:29.445 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:11:29.445 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:11:29.445 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:29.445 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:29 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:11:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:29 smithi064 ceph-mon[22742]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:11:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:29 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:29 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:29 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:29 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:11:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:29 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:11:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:29 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:11:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:29 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:11:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:29 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:11:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:29 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:11:31.466 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:31 smithi059 ceph-mon[21819]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:11:31.466 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:31 smithi059 ceph-mon[26536]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:11:31.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:31 smithi064 ceph-mon[22742]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-26T16:11:33.000 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:11:32.863+0000 7f4c8bc6e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:11:33.001 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:33.002 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:11:33.002 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:33.002 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:11:33.002 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:33.002 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:33.002 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:33.002 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:11:33.003 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:33.004 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:33.005 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:33.005 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:11:33.005 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:33.005 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:33.005 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:33.005 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:33.005 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:33.005 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:11:33.005 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:33.005 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:33.006 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:33.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:33 smithi059 ceph-mon[21819]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:33 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2535024355' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:11:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:33 smithi059 ceph-mon[26536]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:33 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2535024355' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:11:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:33 smithi064 ceph-mon[22742]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:11:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:33 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2535024355' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:11:34.401 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:11:34] "GET /metrics HTTP/1.1" 200 37978 "" "Prometheus/2.33.4" 2024-04-26T16:11:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:34 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:34.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:11:34.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:34.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:34.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:34.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:34.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:34.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:34.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:34.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:34 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:34.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:34.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:34.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:34.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:34.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:11:34.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:34.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:34.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:34.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:34.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:34.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:34.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:34.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:34.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:11:34.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:11:34.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:11:34.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:11:34.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:11:34.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:11:34.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:11:34.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:11:34.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:11:34.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:11:34.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:11:34.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:11:34.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:11:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:11:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:11:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:11:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:34.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:34.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:34.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:34.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:34.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:34.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:34.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:34.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:11:34.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:34.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:34.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:34.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:34.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:34.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:34.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:34.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:34 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.846 INFO:journalctl@ceph.mon.a.smithi059.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:34.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:34.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:34.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:34.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:34.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:34.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:34.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:34.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:11:34.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:34.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:34.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:34.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:34.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:34.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:34.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:34.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:34.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:11:34.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:11:34.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:11:34.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:11:34.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:11:34.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:11:34.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:11:34.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:11:34.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:11:34.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:11:34.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:11:34.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:11:34.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:11:34.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:11:34.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:11:34.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:11:34.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi-foo-smithi059-cazpww 2024-04-26T16:11:34.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: /usr/bin/podman: stderr Error: no such container ceph-b6607c98-03e4-11ef-bc93-c7b262605968-iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:11:34.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:11:34.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:11:34.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:11:34.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:11:34.854 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.854 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:34.854 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:11:34.855 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:11:34.855 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:11:34.855 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:11:34.855 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:11:34.855 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:11:34.856 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:11:34.856 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:11:34.856 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:11:34.856 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:11:34.856 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:11:34.856 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:11:34.857 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:11:34.857 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:11:35.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:35 smithi064 ceph-mon[22742]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T16:11:35.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:35 smithi059 ceph-mon[26536]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T16:11:35.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:35 smithi059 ceph-mon[21819]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T16:11:37.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:37 smithi064 ceph-mon[22742]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T16:11:37.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:37 smithi059 ceph-mon[26536]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T16:11:37.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:37 smithi059 ceph-mon[21819]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T16:11:38.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:38 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:38.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:38 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:38.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:38 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:39.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:39 smithi064 ceph-mon[22742]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T16:11:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:39 smithi059 ceph-mon[26536]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T16:11:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:39 smithi059 ceph-mon[21819]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-26T16:11:41.678 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:41 smithi059 ceph-mon[26536]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-26T16:11:41.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:41 smithi059 ceph-mon[21819]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-26T16:11:41.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:41 smithi064 ceph-mon[22742]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-26T16:11:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:43 smithi064 ceph-mon[22742]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-26T16:11:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:43 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:43.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:43 smithi059 ceph-mon[21819]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-26T16:11:43.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:43 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:43.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:43 smithi059 ceph-mon[26536]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-26T16:11:43.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:43 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:44.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:11:44] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-26T16:11:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:45 smithi064 ceph-mon[22742]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:11:45.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:45 smithi059 ceph-mon[21819]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:11:45.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:45 smithi059 ceph-mon[26536]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:11:47.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:47 smithi064 ceph-mon[22742]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:47.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:47 smithi059 ceph-mon[21819]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:47.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:47 smithi059 ceph-mon[26536]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:49 smithi059 ceph-mon[21819]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:49.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:49 smithi059 ceph-mon[26536]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:49.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:49 smithi064 ceph-mon[22742]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:51.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:51 smithi059 ceph-mon[21819]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:11:51.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:51 smithi059 ceph-mon[26536]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:11:51.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:51 smithi064 ceph-mon[22742]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:11:53.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:53 smithi059 ceph-mon[21819]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:53 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:53 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:53 smithi059 ceph-mon[26536]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:53 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:53 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:53 smithi064 ceph-mon[22742]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:53 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:11:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:53 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:11:54.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:11:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:11:54] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-26T16:11:55.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:55 smithi059 ceph-mon[21819]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:11:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:55 smithi059 ceph-mon[26536]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:11:55.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:55 smithi064 ceph-mon[22742]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:11:57.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:57 smithi059 ceph-mon[21819]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:57.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:57 smithi059 ceph-mon[26536]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:57.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:57 smithi064 ceph-mon[22742]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:59.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:11:59 smithi059 ceph-mon[26536]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:11:59 smithi059 ceph-mon[21819]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:11:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:11:59 smithi064 ceph-mon[22742]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:01.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:01 smithi059 ceph-mon[21819]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:01 smithi059 ceph-mon[26536]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:01 smithi064 ceph-mon[22742]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:03.739 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:03 smithi064 ceph-mon[22742]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:03.739 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:03 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:03 smithi059 ceph-mon[21819]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:03 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:03.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:03 smithi059 ceph-mon[26536]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:03.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:03 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:04.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:12:04] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-26T16:12:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:05 smithi059 ceph-mon[21819]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:05 smithi059 ceph-mon[26536]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:05.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:05 smithi064 ceph-mon[22742]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:07 smithi059 ceph-mon[21819]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:07 smithi059 ceph-mon[26536]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:07.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:07 smithi064 ceph-mon[22742]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:08.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:08.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:08.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:09.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:09 smithi059 ceph-mon[26536]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:09.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:09 smithi059 ceph-mon[21819]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:09.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:09 smithi064 ceph-mon[22742]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:11.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:11 smithi059 ceph-mon[21819]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:11.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:11 smithi059 ceph-mon[26536]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:11 smithi064 ceph-mon[22742]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:13.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:13 smithi059 ceph-mon[26536]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:13.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:13 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:13.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:13 smithi059 ceph-mon[21819]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:13.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:13 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:13.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:13 smithi064 ceph-mon[22742]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:13.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:13 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:14.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:12:14] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-04-26T16:12:15.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:15 smithi059 ceph-mon[26536]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:15 smithi059 ceph-mon[21819]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:15 smithi064 ceph-mon[22742]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:17 smithi059 ceph-mon[21819]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:17.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:17 smithi059 ceph-mon[26536]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:17.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:17 smithi064 ceph-mon[22742]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:19.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:19 smithi059 ceph-mon[21819]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:19 smithi059 ceph-mon[26536]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:19.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:19 smithi064 ceph-mon[22742]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:21.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:21 smithi059 ceph-mon[21819]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:21.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:21 smithi059 ceph-mon[26536]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:21.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:21 smithi064 ceph-mon[22742]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:23.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:23 smithi059 ceph-mon[21819]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:23.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:23 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:23.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:23 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:23 smithi059 ceph-mon[26536]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:23 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:23 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:23.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:23 smithi064 ceph-mon[22742]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:23.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:23 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:23.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:23 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:24.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:12:24] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-26T16:12:25.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:25 smithi059 ceph-mon[21819]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:25.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:25 smithi059 ceph-mon[26536]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:25.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:25 smithi064 ceph-mon[22742]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:27.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:27 smithi059 ceph-mon[21819]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:27.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:27 smithi059 ceph-mon[26536]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:27.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:27 smithi064 ceph-mon[22742]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:29.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:29 smithi059 ceph-mon[26536]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:29.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:29 smithi059 ceph-mon[21819]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:29.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:29 smithi064 ceph-mon[22742]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:31.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:31 smithi059 ceph-mon[26536]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:31.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:31 smithi059 ceph-mon[21819]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:31 smithi064 ceph-mon[22742]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:33.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[21819]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:33.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:33.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:12:33.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[26536]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:33.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:33.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:12:33.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:33 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:33 smithi064 ceph-mon[22742]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:33 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:33 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:12:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:33 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:33 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:33 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:33 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:34.518 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:12:34] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-26T16:12:34.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:12:34.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:12:34.827 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:34.827 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:12:34.827 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:12:34.827 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:12:34.828 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:12:34.828 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:34.828 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:12:34.828 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:34 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:12:34.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:34 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:12:34.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:34 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:12:34.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:34 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:12:34.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:34 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:12:34.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:34 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:12:35.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:35 smithi059 ceph-mon[21819]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:35.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:35 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:12:35.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:35 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:12:35.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:35 smithi059 ceph-mon[26536]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:35.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:35 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:12:35.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:35 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:12:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:35 smithi064 ceph-mon[22742]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:35 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:12:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:35 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:12:36.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:12:36.028+0000 7f4c8bc6e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:12:36.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:12:36.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:12:36.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:12:36.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:12:36.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:12:36.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:12:36.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:12:36.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:36.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:36.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:12:36.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:12:36.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:12:36.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:12:36.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:12:36.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:12:36.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:12:36.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:12:36.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:12:36.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:12:36.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:12:36.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:12:36.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:36.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:37 smithi059 ceph-mon[21819]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:37.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:37 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:12:37.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:12:37.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:12:37.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:12:37.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:12:37.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:12:37.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:12:37.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:12:37.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:37 smithi059 ceph-mon[26536]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:37.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:37 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:12:37.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:12:37.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:12:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:12:37.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:12:37.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:37 smithi064 ceph-mon[22742]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:37 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:12:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:12:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:12:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:12:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:12:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:12:37.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:12:37.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:12:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:12:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:12:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:12:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:12:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:12:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:12:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:12:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:12:37.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:12:38.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:38 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:38.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:38 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:38.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:38 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:39.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:39 smithi059 ceph-mon[26536]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:39 smithi059 ceph-mon[21819]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:39.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:39 smithi064 ceph-mon[22742]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:41 smithi059 ceph-mon[26536]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:41 smithi059 ceph-mon[21819]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:41 smithi064 ceph-mon[22742]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:43.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:43 smithi059 ceph-mon[26536]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:43.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:43 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:43 smithi059 ceph-mon[21819]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:43 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:43.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:43 smithi064 ceph-mon[22742]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:43.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:43 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:44.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:12:44] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-26T16:12:45.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:45 smithi059 ceph-mon[21819]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:45.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:45 smithi059 ceph-mon[26536]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:45.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:45 smithi064 ceph-mon[22742]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:47.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:47 smithi059 ceph-mon[21819]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:47.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:47 smithi059 ceph-mon[26536]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:47.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:47 smithi064 ceph-mon[22742]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:49.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:49 smithi059 ceph-mon[21819]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:49.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:49 smithi059 ceph-mon[26536]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:49.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:49 smithi064 ceph-mon[22742]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:51.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:51 smithi059 ceph-mon[21819]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:51.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:51 smithi059 ceph-mon[26536]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:51.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:51 smithi064 ceph-mon[22742]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:53.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:53 smithi059 ceph-mon[21819]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:53 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:53 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:53 smithi059 ceph-mon[26536]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:53 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:53 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:53 smithi064 ceph-mon[22742]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:53 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:12:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:53 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:12:54.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:12:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:12:54] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-26T16:12:55.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:55 smithi059 ceph-mon[26536]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:55 smithi059 ceph-mon[21819]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:55.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:55 smithi064 ceph-mon[22742]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:12:57.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:57 smithi059 ceph-mon[21819]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:57.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:57 smithi059 ceph-mon[26536]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:57.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:57 smithi064 ceph-mon[22742]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:59.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:12:59 smithi059 ceph-mon[26536]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:12:59 smithi059 ceph-mon[21819]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:12:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:12:59 smithi064 ceph-mon[22742]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:01.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:01 smithi059 ceph-mon[21819]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:01 smithi059 ceph-mon[26536]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:01.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:01 smithi064 ceph-mon[22742]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:03.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:03 smithi059 ceph-mon[26536]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:03.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:03 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:03 smithi059 ceph-mon[21819]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:03 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:03.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:03 smithi064 ceph-mon[22742]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:03.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:03 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:04.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:13:04] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-26T16:13:05.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:05 smithi059 ceph-mon[26536]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:05 smithi059 ceph-mon[21819]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:05.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:05 smithi064 ceph-mon[22742]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:07.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:07 smithi059 ceph-mon[26536]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:07 smithi059 ceph-mon[21819]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:07.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:07 smithi064 ceph-mon[22742]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:08.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:09.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:09.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:09.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:09 smithi064 ceph-mon[22742]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:10.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:09 smithi059 ceph-mon[26536]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:10.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:09 smithi059 ceph-mon[21819]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:11 smithi064 ceph-mon[22742]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:12.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:11 smithi059 ceph-mon[26536]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:12.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:11 smithi059 ceph-mon[21819]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:13.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:13 smithi064 ceph-mon[22742]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:13.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:13 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:14.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:13 smithi059 ceph-mon[26536]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:14.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:13 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:14.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:13 smithi059 ceph-mon[21819]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:14.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:13 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:14.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:13:14] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-26T16:13:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:15 smithi064 ceph-mon[22742]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:16.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:15 smithi059 ceph-mon[21819]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:16.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:15 smithi059 ceph-mon[26536]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:17.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:17 smithi064 ceph-mon[22742]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:18.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:17 smithi059 ceph-mon[21819]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:18.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:17 smithi059 ceph-mon[26536]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:19.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:19 smithi064 ceph-mon[22742]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:20.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:19 smithi059 ceph-mon[21819]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:20.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:19 smithi059 ceph-mon[26536]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:21.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:21 smithi064 ceph-mon[22742]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:22.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:21 smithi059 ceph-mon[21819]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:22.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:21 smithi059 ceph-mon[26536]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:23.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:23 smithi064 ceph-mon[22742]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:23.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:23 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:23.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:23 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:24.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:23 smithi059 ceph-mon[26536]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:24.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:23 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:24.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:23 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:24.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:23 smithi059 ceph-mon[21819]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:24.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:23 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:24.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:23 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:24.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:13:24] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-04-26T16:13:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:25 smithi064 ceph-mon[22742]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:26.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:25 smithi059 ceph-mon[21819]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:26.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:25 smithi059 ceph-mon[26536]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:27 smithi064 ceph-mon[22742]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:28.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:27 smithi059 ceph-mon[21819]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:28.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:27 smithi059 ceph-mon[26536]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:29.884 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:29 smithi059 ceph-mon[21819]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:29.884 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:29 smithi059 ceph-mon[26536]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:29.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:29 smithi064 ceph-mon[22742]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:31 smithi064 ceph-mon[22742]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:32.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:31 smithi059 ceph-mon[21819]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:32.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:31 smithi059 ceph-mon[26536]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:33 smithi064 ceph-mon[22742]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:33 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:34.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:33 smithi059 ceph-mon[21819]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:34.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:33 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:34.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:33 smithi059 ceph-mon[26536]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:34.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:33 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:34.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:13:34] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-04-26T16:13:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:35 smithi064 ceph-mon[22742]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:36.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:35 smithi059 ceph-mon[21819]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:35 smithi059 ceph-mon[26536]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:36 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:13:37.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:36 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:13:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:36 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:13:37.765 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[26536]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:37.765 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:13:37.765 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:13:37.765 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:13:37.765 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:13:37.765 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:13:37.765 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[21819]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:37.765 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:13:37.765 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:13:37.765 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:13:37.766 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:13:37.766 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:37 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:13:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:37 smithi064 ceph-mon[22742]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:37 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:13:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:37 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:13:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:37 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' 2024-04-26T16:13:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:37 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:13:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:37 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:13:38.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:13:38.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:13:38.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:38.727 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:13:38.727 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:13:38.728 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:38.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:38 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:13:38.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:38 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:13:38.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:38 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:39.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:13:38.727+0000 7f4c8bc6e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:13:39.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:13:39.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:13:39.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:13:39.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:13:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:13:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:13:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:13:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:13:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:13:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:13:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:13:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:13:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:13:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:13:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:13:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:39 smithi064 ceph-mon[22742]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:40.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:39 smithi059 ceph-mon[26536]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:40.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:39 smithi059 ceph-mon[21819]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:40 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:13:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:13:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:13:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:13:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:13:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:13:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:13:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:13:40.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:13:40.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:13:40.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:13:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:13:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:13:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:13:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:13:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:13:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:13:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:13:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:13:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:40.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:41.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:40 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:13:41.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:13:41.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:13:41.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:13:41.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:13:41.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:41.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:13:41.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:13:41.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:41.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:13:41.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:41.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:41.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:40 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:13:41.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:13:41.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:13:41.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:13:41.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:13:41.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:13:41.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:13:41.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:13:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:13:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:13:41.093 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:13:41.093 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:13:41.093 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:13:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:41 smithi064 ceph-mon[22742]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:42.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:41 smithi059 ceph-mon[26536]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:41 smithi059 ceph-mon[21819]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:43.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:43 smithi064 ceph-mon[22742]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:43.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:43 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:44.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:43 smithi059 ceph-mon[21819]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:43 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:43 smithi059 ceph-mon[26536]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:43 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:44.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:13:44] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-04-26T16:13:45.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:45 smithi064 ceph-mon[22742]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:46.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:45 smithi059 ceph-mon[21819]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:45 smithi059 ceph-mon[26536]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:47.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:47 smithi064 ceph-mon[22742]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:48.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:47 smithi059 ceph-mon[26536]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:47 smithi059 ceph-mon[21819]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:49.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:49 smithi064 ceph-mon[22742]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:50.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:49 smithi059 ceph-mon[26536]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:50.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:49 smithi059 ceph-mon[21819]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:51.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:51 smithi064 ceph-mon[22742]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:52.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:51 smithi059 ceph-mon[26536]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:51 smithi059 ceph-mon[21819]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:53 smithi064 ceph-mon[22742]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:53 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:53 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:54.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:53 smithi059 ceph-mon[26536]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:54.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:53 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:54.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:53 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:54.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:53 smithi059 ceph-mon[21819]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:54.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:53 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:13:54.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:53 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:13:54.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:13:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:13:54] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.33.4" 2024-04-26T16:13:55.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:55 smithi064 ceph-mon[22742]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:56.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:55 smithi059 ceph-mon[21819]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:56.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:55 smithi059 ceph-mon[26536]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:13:57.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:57 smithi064 ceph-mon[22742]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:58.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:57 smithi059 ceph-mon[21819]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:58.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:57 smithi059 ceph-mon[26536]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:59.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:13:59 smithi059 ceph-mon[26536]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:59.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:13:59 smithi059 ceph-mon[21819]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:13:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:13:59 smithi064 ceph-mon[22742]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:01 smithi064 ceph-mon[22742]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:02.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:01 smithi059 ceph-mon[21819]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:02.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:01 smithi059 ceph-mon[26536]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:03.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:03 smithi059 ceph-mon[21819]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:03.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:03 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:03 smithi059 ceph-mon[26536]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:03 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:03 smithi064 ceph-mon[22742]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:03 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:04.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:14:04] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.33.4" 2024-04-26T16:14:05.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:05 smithi059 ceph-mon[21819]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:05.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:05 smithi059 ceph-mon[26536]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:05.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:05 smithi064 ceph-mon[22742]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:07.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:07 smithi064 ceph-mon[22742]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:07 smithi059 ceph-mon[26536]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:07 smithi059 ceph-mon[21819]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:08 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:08.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:08 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:08.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:08 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:09 smithi064 ceph-mon[22742]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:09.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:09 smithi059 ceph-mon[26536]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:09.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:09 smithi059 ceph-mon[21819]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:11 smithi059 ceph-mon[21819]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:11 smithi059 ceph-mon[26536]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:11 smithi064 ceph-mon[22742]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:12.027 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:14:13.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:13 smithi059 ceph-mon[26536]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:13.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:13 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:13 smithi059 ceph-mon[21819]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:13 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:13.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:13 smithi064 ceph-mon[22742]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:13 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:14.185 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (9m) 3m ago 16m 24.4M - 0.23.0 ba2b418f427c ca6d4a23fcc2 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (9m) 2m ago 16m 55.6M - 8.3.5 dad864ee21e9 0d16158d9791 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 running (8m) 3m ago 16m 55.5M - 3.5 e1d6a67b021e 65a32364fe46 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283 running (11m) 2m ago 21m 462M - 19.0.0-3305-ga430f9ed 031ba2747cfd 03edb3b65e45 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (6m) 3m ago 23m 490M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (23m) 3m ago 23m 59.6M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (21m) 2m ago 21m 51.0M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (21m) 3m ago 21m 48.1M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (9m) 3m ago 17m 18.7M - 1.3.1 1dbe0e931976 9b14cfbf9c80 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (9m) 2m ago 17m 17.3M - 1.3.1 1dbe0e931976 fb96f1dc25f3 2024-04-26T16:14:14.186 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (21m) 3m ago 21m 53.1M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:14:14.187 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (20m) 3m ago 20m 52.9M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:14:14.187 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (20m) 3m ago 20m 51.6M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:14:14.187 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (19m) 3m ago 19m 52.2M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:14:14.187 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (19m) 2m ago 19m 52.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:14:14.187 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (19m) 2m ago 19m 53.3M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:14:14.187 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (18m) 2m ago 18m 50.3M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:14:14.187 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (18m) 2m ago 18m 52.7M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:14:14.187 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (2m) 2m ago 17m 44.3M - 2.33.4 514e6a882f6e 25fb46b85a95 2024-04-26T16:14:14.187 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (16m) 3m ago 16m 73.1M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:14:14.187 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (16m) 2m ago 16m 73.5M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:14:14.187 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:14:14] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-26T16:14:14.868 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions' 2024-04-26T16:14:15.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:15 smithi059 ceph-mon[21819]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:15.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:15 smithi059 ceph-mon[21819]: from='client.25198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:14:15.447 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:15 smithi059 ceph-mon[26536]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:15.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:15 smithi059 ceph-mon[26536]: from='client.25198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:14:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:15 smithi064 ceph-mon[22742]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:15 smithi064 ceph-mon[22742]: from='client.25198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:14:16.967 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:14:16.968 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:14:16.969 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:14:16.969 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:14:16.969 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T16:14:16.969 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:14:16.969 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:14:16.969 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:14:17.550 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:17 smithi059 ceph-mon[26536]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:17.551 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:17 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3257259748' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:14:17.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:17 smithi059 ceph-mon[21819]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:17.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:17 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3257259748' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:14:17.576 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph -s' 2024-04-26T16:14:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:17 smithi064 ceph-mon[22742]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:17 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3257259748' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:14:19.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:19 smithi059 ceph-mon[21819]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:19.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:19 smithi059 ceph-mon[26536]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:19.666 INFO:teuthology.orchestra.run.smithi059.stdout: cluster: 2024-04-26T16:14:19.666 INFO:teuthology.orchestra.run.smithi059.stdout: id: b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:14:19.666 INFO:teuthology.orchestra.run.smithi059.stdout: health: HEALTH_OK 2024-04-26T16:14:19.666 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:14:19.666 INFO:teuthology.orchestra.run.smithi059.stdout: services: 2024-04-26T16:14:19.666 INFO:teuthology.orchestra.run.smithi059.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-04-26T16:14:19.666 INFO:teuthology.orchestra.run.smithi059.stdout: mgr: y(active, since 3m), standbys: x 2024-04-26T16:14:19.666 INFO:teuthology.orchestra.run.smithi059.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-04-26T16:14:19.666 INFO:teuthology.orchestra.run.smithi059.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-26T16:14:19.667 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:14:19.667 INFO:teuthology.orchestra.run.smithi059.stdout: data: 2024-04-26T16:14:19.667 INFO:teuthology.orchestra.run.smithi059.stdout: pools: 6 pools, 161 pgs 2024-04-26T16:14:19.667 INFO:teuthology.orchestra.run.smithi059.stdout: objects: 209 objects, 585 KiB 2024-04-26T16:14:19.667 INFO:teuthology.orchestra.run.smithi059.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:14:19.667 INFO:teuthology.orchestra.run.smithi059.stdout: pgs: 161 active+clean 2024-04-26T16:14:19.667 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:14:19.667 INFO:teuthology.orchestra.run.smithi059.stdout: io: 2024-04-26T16:14:19.667 INFO:teuthology.orchestra.run.smithi059.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-26T16:14:19.667 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-26T16:14:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:19 smithi064 ceph-mon[22742]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:20.313 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:14:20.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:20 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2141818969' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:14:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:20 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2141818969' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:14:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:20 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2141818969' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T16:14:21.638 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:21 smithi059 ceph-mon[21819]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:21.639 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:21 smithi059 ceph-mon[26536]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:21 smithi064 ceph-mon[22742]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:22.413 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_OK 2024-04-26T16:14:23.065 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-26T16:14:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:23 smithi064 ceph-mon[22742]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:23 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/159583129' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:14:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:23 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:23 smithi064 ceph-mon[22742]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:23.691 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:23 smithi059 ceph-mon[26536]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:23.691 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:23 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/159583129' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:14:23.691 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:23 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:23.691 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:23 smithi059 ceph-mon[26536]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:23.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:23 smithi059 ceph-mon[21819]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:23.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:23 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/159583129' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:14:23.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:23 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:23.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:23 smithi059 ceph-mon[21819]: from='mgr.25108 172.21.15.59:0/968472417' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:24.319 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:14:24] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-04-26T16:14:25.117 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:14:25.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:25 smithi064 ceph-mon[22742]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:25.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:25 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/971401097' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:14:25.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:25 smithi059 ceph-mon[26536]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:25.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:25 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/971401097' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:14:25.711 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:25 smithi059 ceph-mon[21819]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:14:25.711 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:25 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/971401097' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:14:25.734 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph mgr fail' 2024-04-26T16:14:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:27 smithi059 ceph-mon[21819]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:27 smithi059 ceph-mon[26536]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:27 smithi064 ceph-mon[22742]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:28.634 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3720389607' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:14:28.634 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:14:28.634 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-mon[21819]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T16:14:28.634 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3720389607' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:14:28.635 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:14:28.635 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-mon[26536]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T16:14:28.635 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:28.393+0000 7f4cfa6e1700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T16:14:28.635 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ignoring --setuser ceph since I am not root 2024-04-26T16:14:28.635 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ignoring --setgroup ceph since I am not root 2024-04-26T16:14:28.635 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-mgr[60319]: -- 172.21.15.59:0/908734802 <== mon.1 v2:172.21.15.59:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560fce37e5a0 con 0x560fce353000 2024-04-26T16:14:28.635 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:28.585+0000 7f4788a6a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T16:14:28.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:28 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3720389607' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:14:28.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:28 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-26T16:14:28.636 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:28 smithi064 ceph-mon[22742]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T16:14:28.636 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:28] ENGINE Bus STOPPING 2024-04-26T16:14:28.636 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:28] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:14:28.636 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:28] ENGINE Bus STOPPED 2024-04-26T16:14:28.636 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:28] ENGINE Bus STARTING 2024-04-26T16:14:28.636 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:28] ENGINE Serving on http://:::9283 2024-04-26T16:14:28.636 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:28] ENGINE Bus STARTED 2024-04-26T16:14:28.895 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:28.782+0000 7f4788a6a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:14:29.046 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'sleep 180' 2024-04-26T16:14:29.236 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:28.894+0000 7f4788a6a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T16:14:29.236 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:29.235+0000 7f4788a6a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T16:14:29.510 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:29.341+0000 7f4788a6a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T16:14:29.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-26T16:14:29.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: mgrmap e33: x(active, starting, since 0.626033s) 2024-04-26T16:14:29.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:14:29.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:14:29.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:14:29.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:14:29.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:14:29.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:14:29.511 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: Manager daemon x is now available 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:14:29.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:14:29.513 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:14:29.513 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:14:29.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-26T16:14:29.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: mgrmap e33: x(active, starting, since 0.626033s) 2024-04-26T16:14:29.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:14:29.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:14:29.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:14:29.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:14:29.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:14:29.513 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: Manager daemon x is now available 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:29.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:14:29.515 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:14:29.515 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:14:29.515 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:29 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:14:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-26T16:14:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: mgrmap e33: x(active, starting, since 0.626033s) 2024-04-26T16:14:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:14:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:14:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:14:29.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:14:29.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: Manager daemon x is now available 2024-04-26T16:14:29.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:14:29.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:29.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:14:29.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:14:29.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:14:29.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:29 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T16:14:30.498 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:30 smithi059 ceph-mon[26536]: mgrmap e34: x(active, since 1.63373s) 2024-04-26T16:14:30.499 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:30 smithi059 ceph-mon[26536]: [26/Apr/2024:16:14:29] ENGINE Bus STARTING 2024-04-26T16:14:30.499 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:30 smithi059 ceph-mon[26536]: [26/Apr/2024:16:14:29] ENGINE Serving on https://172.21.15.64:7150 2024-04-26T16:14:30.499 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:30 smithi059 ceph-mon[21819]: mgrmap e34: x(active, since 1.63373s) 2024-04-26T16:14:30.499 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:30 smithi059 ceph-mon[21819]: [26/Apr/2024:16:14:29] ENGINE Bus STARTING 2024-04-26T16:14:30.499 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:30 smithi059 ceph-mon[21819]: [26/Apr/2024:16:14:29] ENGINE Serving on https://172.21.15.64:7150 2024-04-26T16:14:30.682 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:30 smithi064 ceph-mon[22742]: mgrmap e34: x(active, since 1.63373s) 2024-04-26T16:14:30.683 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:30 smithi064 ceph-mon[22742]: [26/Apr/2024:16:14:29] ENGINE Bus STARTING 2024-04-26T16:14:30.683 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:30 smithi064 ceph-mon[22742]: [26/Apr/2024:16:14:29] ENGINE Serving on https://172.21.15.64:7150 2024-04-26T16:14:30.751 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:30.659+0000 7f4788a6a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T16:14:31.037 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:30.846+0000 7f4788a6a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T16:14:31.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:31.036+0000 7f4788a6a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T16:14:31.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:31 smithi064 ceph-mon[22742]: [26/Apr/2024:16:14:29] ENGINE Serving on http://172.21.15.64:8765 2024-04-26T16:14:31.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:31 smithi064 ceph-mon[22742]: [26/Apr/2024:16:14:29] ENGINE Bus STARTED 2024-04-26T16:14:31.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:31 smithi064 ceph-mon[22742]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:14:31.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:31 smithi064 ceph-mon[22742]: mgrmap e35: x(active, since 2s) 2024-04-26T16:14:31.697 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:31.364+0000 7f4788a6a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T16:14:31.697 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:31.476+0000 7f4788a6a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T16:14:31.697 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-mon[21819]: [26/Apr/2024:16:14:29] ENGINE Serving on http://172.21.15.64:8765 2024-04-26T16:14:31.697 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-mon[21819]: [26/Apr/2024:16:14:29] ENGINE Bus STARTED 2024-04-26T16:14:31.697 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-mon[21819]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:14:31.697 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-mon[21819]: mgrmap e35: x(active, since 2s) 2024-04-26T16:14:31.698 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-mon[26536]: [26/Apr/2024:16:14:29] ENGINE Serving on http://172.21.15.64:8765 2024-04-26T16:14:31.698 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-mon[26536]: [26/Apr/2024:16:14:29] ENGINE Bus STARTED 2024-04-26T16:14:31.698 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-mon[26536]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:14:31.698 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-mon[26536]: mgrmap e35: x(active, since 2s) 2024-04-26T16:14:32.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:31.696+0000 7f4788a6a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T16:14:32.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:31.911+0000 7f4788a6a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:14:32.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:32.595+0000 7f4788a6a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T16:14:33.259 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:32.833+0000 7f4788a6a200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-04-26T16:14:33.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:33.304+0000 7f4788a6a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T16:14:33.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:33.412+0000 7f4788a6a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T16:14:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-mon[26536]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:14:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-mon[26536]: mgrmap e36: x(active, since 4s) 2024-04-26T16:14:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-mon[21819]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:14:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-mon[21819]: mgrmap e36: x(active, since 4s) 2024-04-26T16:14:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:33.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:33.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:33.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:33 smithi064 ceph-mon[22742]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:14:33.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:33 smithi064 ceph-mon[22742]: mgrmap e36: x(active, since 4s) 2024-04-26T16:14:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:33 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:33 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:33 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:34.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:33.810+0000 7f4788a6a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T16:14:34.569 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:34.201+0000 7f4788a6a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T16:14:34.570 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:34.300+0000 7f4788a6a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T16:14:34.570 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:34.569+0000 7f4788a6a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T16:14:34.822 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:34.716+0000 7f4788a6a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T16:14:34.822 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:34.821+0000 7f4788a6a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T16:14:35.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:35.170+0000 7f4788a6a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T16:14:35.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:35.274+0000 7f4788a6a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T16:14:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-mon[21819]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:14:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-mon[26536]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:14:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:35 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:35 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:35 smithi064 ceph-mon[22742]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:14:35.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:35 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:35.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:35 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:36.297 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:35 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:35.944+0000 7f4788a6a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T16:14:36.297 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:36.188+0000 7f4788a6a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T16:14:36.297 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:14:36] ENGINE Bus STARTING 2024-04-26T16:14:36.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CherryPy Checker: 2024-04-26T16:14:36.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: The Application mounted at '' has an empty config. 2024-04-26T16:14:36.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:14:36.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:14:36] ENGINE Serving on http://:::9283 2024-04-26T16:14:36.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:14:36] ENGINE Bus STARTED 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: Standby manager daemon y started 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T16:14:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:36 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:14:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:14:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:14:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:14:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:14:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: Standby manager daemon y started 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:14:37.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: Standby manager daemon y started 2024-04-26T16:14:37.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T16:14:37.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:14:37.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T16:14:37.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:36 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:14:37.929 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:14:37.929 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:14:37.929 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:14:37.929 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:37.930 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:37 smithi064 ceph-mon[22742]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[21819]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:14:38.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:14:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-26T16:14:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:14:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T16:14:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:38.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:37 smithi059 ceph-mon[26536]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:14:38.922 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:38 smithi064 ceph-mon[22742]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:14:38.922 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:38 smithi064 ceph-mon[22742]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:14:39.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:38 smithi059 ceph-mon[21819]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:14:39.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:38 smithi059 ceph-mon[21819]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:14:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:38 smithi059 ceph-mon[26536]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:14:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:38 smithi059 ceph-mon[26536]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:14:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:39 smithi064 ceph-mon[22742]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:14:40.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:39 smithi059 ceph-mon[21819]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:14:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:39 smithi059 ceph-mon[26536]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:14:40.417 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 systemd[1]: Stopping Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:14:40.417 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 bash[54658]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-26T16:14:40.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.417Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T16:14:40.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-26T16:14:40.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-26T16:14:40.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-26T16:14:40.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-26T16:14:40.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-26T16:14:40.688 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T16:14:40.688 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-26T16:14:40.688 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T16:14:40.688 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T16:14:40.688 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-26T16:14:40.688 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[51743]: ts=2024-04-26T16:14:40.418Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-26T16:14:40.688 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 podman[54669]: 2024-04-26 16:14:40.428212938 +0000 UTC m=+0.141438661 container died 25fb46b85a951e02079fcb4c41c421f1555909da2b939df37ae2ea5c2bc0e4fb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:14:40.976 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 podman[54669]: 2024-04-26 16:14:40.72555356 +0000 UTC m=+0.438779288 container cleanup 25fb46b85a951e02079fcb4c41c421f1555909da2b939df37ae2ea5c2bc0e4fb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:14:40.976 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 bash[54669]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T16:14:40.976 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 bash[54697]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-26T16:14:41.243 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 podman[54683]: 2024-04-26 16:14:40.976710137 +0000 UTC m=+0.549723812 container remove 25fb46b85a951e02079fcb4c41c421f1555909da2b939df37ae2ea5c2bc0e4fb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:14:41.243 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T16:14:41.244 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 systemd[1]: Stopped Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:14:41.244 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.403s CPU time. 2024-04-26T16:14:41.244 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:40 smithi064 systemd[1]: Starting Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:14:41.244 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 podman[54763]: 2024-04-26 16:14:41.119509321 +0000 UTC m=+0.020582465 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-26T16:14:41.244 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 podman[54763]: 2024-04-26 16:14:41.24439138 +0000 UTC m=+0.145464554 container create 2614c60ef06fa639bb199e2cbd3b28f4af74d935b9c89e34dae28538afbf4896 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:14:41.529 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 podman[54763]: 2024-04-26 16:14:41.455010652 +0000 UTC m=+0.356083787 container init 2614c60ef06fa639bb199e2cbd3b28f4af74d935b9c89e34dae28538afbf4896 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:14:41.529 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 podman[54763]: 2024-04-26 16:14:41.457738562 +0000 UTC m=+0.358811697 container start 2614c60ef06fa639bb199e2cbd3b28f4af74d935b9c89e34dae28538afbf4896 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.481Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.481Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.481Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.481Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi064 (none))" 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.481Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.481Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.483Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.483Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.485Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.486Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.488Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.281815ms 2024-04-26T16:14:41.530 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.488Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T16:14:41.531 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 bash[54763]: 2614c60ef06fa639bb199e2cbd3b28f4af74d935b9c89e34dae28538afbf4896 2024-04-26T16:14:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-mon[22742]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:14:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:14:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:14:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:14:41.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:41] ENGINE Bus STOPPING 2024-04-26T16:14:41.938 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:14:41.938 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:41] ENGINE Bus STOPPED 2024-04-26T16:14:41.938 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:41] ENGINE Bus STARTING 2024-04-26T16:14:41.938 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:41] ENGINE Serving on http://:::9283 2024-04-26T16:14:41.938 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:14:41] ENGINE Bus STARTED 2024-04-26T16:14:41.938 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 systemd[1]: Started Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:14:41.938 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.691Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-04-26T16:14:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[21819]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:14:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:14:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:14:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:14:42.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[26536]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:14:42.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:42.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:42.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:14:42.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:14:42.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:42.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:41 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:14:42.221 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:41 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:41.962Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-04-26T16:14:42.221 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:42.186Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-04-26T16:14:42.221 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:42.190Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-04-26T16:14:42.221 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:42.190Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-04-26T16:14:42.221 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:42.190Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=57.379µs wal_replay_duration=702.158851ms total_replay_duration=703.521489ms 2024-04-26T16:14:42.221 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:42.194Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T16:14:42.221 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:42.194Z caller=main.go:947 level=info msg="TSDB started" 2024-04-26T16:14:42.222 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:42.194Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T16:14:42.222 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:42.221Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.50862ms db_storage=909ns remote_storage=2.279µs web_handler=540ns query_engine=1.09µs scrape=609.183µs scrape_sd=154.929µs notify=20.071µs notify_sd=20.075µs rules=25.166141ms 2024-04-26T16:14:42.222 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:14:42.221Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-26T16:14:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:14:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:42 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:14:43.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:42 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:14:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:42 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:14:43.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:42 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:14:43.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:42 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064:9095"}]: dispatch 2024-04-26T16:14:44.026 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:43 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:44.026 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:43 smithi064 ceph-mon[22742]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:14:44.026 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:43 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:44.026 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:43 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:43 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:43 smithi059 ceph-mon[26536]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:14:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:43 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:43 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:43 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:43 smithi059 ceph-mon[21819]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:14:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:43 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:43 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[26536]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:14:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:14:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:14:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:46.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[21819]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:14:46.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:46.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:46.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:14:46.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:14:46.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:45 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:45 smithi064 ceph-mon[22742]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:14:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:45 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:45 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:45 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:14:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:45 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:14:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:45 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:14:47.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:46 smithi059 ceph-mon[26536]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:47.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:46 smithi059 ceph-mon[26536]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:46 smithi059 ceph-mon[21819]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:46 smithi059 ceph-mon[21819]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:47.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:46 smithi064 ceph-mon[22742]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:47.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:46 smithi064 ceph-mon[22742]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:48.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:48 smithi064 ceph-mon[22742]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:48.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:48 smithi059 ceph-mon[26536]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:48 smithi059 ceph-mon[21819]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:51.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:50 smithi059 ceph-mon[21819]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:51.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:50 smithi059 ceph-mon[26536]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:50 smithi064 ceph-mon[22742]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:53.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:52 smithi059 ceph-mon[26536]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:53.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:52 smithi059 ceph-mon[21819]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:53.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:52 smithi064 ceph-mon[22742]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:14:53.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:14:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:14:52] "GET /metrics HTTP/1.1" 200 38047 "" "Prometheus/2.33.4" 2024-04-26T16:14:54.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:53 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:54.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:53 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:54.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:53 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:14:55.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:54 smithi059 ceph-mon[26536]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 565 B/s rd, 0 op/s 2024-04-26T16:14:55.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:54 smithi059 ceph-mon[21819]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 565 B/s rd, 0 op/s 2024-04-26T16:14:55.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:54 smithi064 ceph-mon[22742]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 565 B/s rd, 0 op/s 2024-04-26T16:14:57.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:56 smithi059 ceph-mon[21819]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:14:57.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:56 smithi059 ceph-mon[26536]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:14:57.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:56 smithi064 ceph-mon[22742]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:14:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:58 smithi064 ceph-mon[22742]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:58.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:58 smithi059 ceph-mon[21819]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:58.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:58 smithi059 ceph-mon[26536]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:14:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:14:59 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:59.734 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:14:59 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:14:59.734 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:14:59 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:00.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:00 smithi064 ceph-mon[22742]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:00.746 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:00 smithi059 ceph-mon[21819]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:00.746 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:00 smithi059 ceph-mon[26536]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:03.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:02 smithi059 ceph-mon[21819]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:02 smithi059 ceph-mon[26536]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:02 smithi064 ceph-mon[22742]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:03.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:15:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:15:02] "GET /metrics HTTP/1.1" 200 38051 "" "Prometheus/2.33.4" 2024-04-26T16:15:04.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:03 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:04.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:03 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:04.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:03 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:05.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:04 smithi059 ceph-mon[26536]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:04 smithi059 ceph-mon[21819]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:05.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:04 smithi064 ceph-mon[22742]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:07.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:06 smithi059 ceph-mon[26536]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:07.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:06 smithi059 ceph-mon[21819]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:07.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:06 smithi064 ceph-mon[22742]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:08 smithi064 ceph-mon[22742]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:08.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:08 smithi059 ceph-mon[26536]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:08.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:08 smithi059 ceph-mon[21819]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:11.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:10 smithi059 ceph-mon[21819]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:11.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:10 smithi059 ceph-mon[26536]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:11.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:10 smithi064 ceph-mon[22742]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:13.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:12 smithi059 ceph-mon[21819]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:13.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:12 smithi059 ceph-mon[26536]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:13.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:12 smithi064 ceph-mon[22742]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:13.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:15:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:15:12] "GET /metrics HTTP/1.1" 200 38051 "" "Prometheus/2.33.4" 2024-04-26T16:15:14.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:13 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:14.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:13 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:14.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:13 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:14.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:13 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:14.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:13 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:14.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:13 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:15.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:14 smithi059 ceph-mon[21819]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:15.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:14 smithi059 ceph-mon[26536]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:15.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:14 smithi064 ceph-mon[22742]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:16 smithi059 ceph-mon[21819]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:17.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:16 smithi059 ceph-mon[26536]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:17.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:16 smithi064 ceph-mon[22742]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:18 smithi064 ceph-mon[22742]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:18.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:18 smithi059 ceph-mon[21819]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:18.994 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:18 smithi059 ceph-mon[26536]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:21.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:20 smithi059 ceph-mon[21819]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:21.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:20 smithi059 ceph-mon[26536]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:21.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:20 smithi064 ceph-mon[22742]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:23.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:22 smithi059 ceph-mon[21819]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:23.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:22 smithi059 ceph-mon[26536]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:23.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:22 smithi064 ceph-mon[22742]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:23.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:15:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:15:22] "GET /metrics HTTP/1.1" 200 38053 "" "Prometheus/2.33.4" 2024-04-26T16:15:24.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:23 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:24.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:23 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:24.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:23 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:25.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:24 smithi059 ceph-mon[21819]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:24 smithi059 ceph-mon[26536]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:25.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:24 smithi064 ceph-mon[22742]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:27.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:26 smithi059 ceph-mon[21819]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:27.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:26 smithi059 ceph-mon[26536]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:27.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:26 smithi064 ceph-mon[22742]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:28 smithi064 ceph-mon[22742]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:28.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:28 smithi059 ceph-mon[21819]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:28 smithi059 ceph-mon[26536]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:29 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:29.734 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:29 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:29.735 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:29 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:30 smithi064 ceph-mon[22742]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:30.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:30 smithi059 ceph-mon[21819]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:30 smithi059 ceph-mon[26536]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:33.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:32 smithi059 ceph-mon[21819]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:33.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:32 smithi059 ceph-mon[26536]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:33.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:32 smithi064 ceph-mon[22742]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:33.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:15:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:15:32] "GET /metrics HTTP/1.1" 200 38051 "" "Prometheus/2.33.4" 2024-04-26T16:15:34.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:33 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:34.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:33 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:34.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:33 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:35.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:34 smithi059 ceph-mon[21819]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:35.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:34 smithi059 ceph-mon[26536]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:35.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:34 smithi064 ceph-mon[22742]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:37.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:36 smithi059 ceph-mon[21819]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:36 smithi059 ceph-mon[26536]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:37.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:36 smithi064 ceph-mon[22742]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:38 smithi064 ceph-mon[22742]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:38.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:38 smithi059 ceph-mon[21819]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:38.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:38 smithi059 ceph-mon[26536]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:41.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:40 smithi059 ceph-mon[21819]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:41.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:40 smithi059 ceph-mon[26536]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:40 smithi064 ceph-mon[22742]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:42 smithi059 ceph-mon[26536]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:43.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:42 smithi059 ceph-mon[21819]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:43.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:42 smithi064 ceph-mon[22742]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:43.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:15:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:15:42] "GET /metrics HTTP/1.1" 200 38051 "" "Prometheus/2.33.4" 2024-04-26T16:15:44.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:43 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:43 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:43 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:43 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:43 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:43 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:45.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:44 smithi059 ceph-mon[26536]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:44 smithi059 ceph-mon[21819]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:45.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:44 smithi064 ceph-mon[22742]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:47.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:46 smithi064 ceph-mon[22742]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:47.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:46 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:15:47.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:46 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:15:47.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:46 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:15:47.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:46 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:15:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:46 smithi059 ceph-mon[26536]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:46 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:15:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:46 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:15:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:46 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:15:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:46 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:15:47.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:46 smithi059 ceph-mon[21819]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:15:47.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:46 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:15:47.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:46 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:15:47.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:46 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:15:47.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:46 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:15:48.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:47 smithi064 ceph-mon[22742]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-04-26T16:15:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:47 smithi064 ceph-mon[22742]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:48.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:47 smithi059 ceph-mon[26536]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-04-26T16:15:48.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:47 smithi059 ceph-mon[26536]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:48.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:47 smithi059 ceph-mon[21819]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-04-26T16:15:48.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:47 smithi059 ceph-mon[21819]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:50.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:49 smithi064 ceph-mon[22742]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-04-26T16:15:50.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:49 smithi059 ceph-mon[26536]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-04-26T16:15:50.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:49 smithi059 ceph-mon[21819]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-04-26T16:15:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:51 smithi064 ceph-mon[22742]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:52.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:51 smithi059 ceph-mon[26536]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:52.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:51 smithi059 ceph-mon[21819]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:53.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:15:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:15:52] "GET /metrics HTTP/1.1" 200 38050 "" "Prometheus/2.33.4" 2024-04-26T16:15:53.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:53 smithi064 ceph-mon[22742]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:53.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:53 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:53 smithi059 ceph-mon[21819]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:53 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:53 smithi059 ceph-mon[26536]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:53 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:15:55.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:55 smithi064 ceph-mon[22742]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:56.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:55 smithi059 ceph-mon[21819]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:56.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:55 smithi059 ceph-mon[26536]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:15:57.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:57 smithi064 ceph-mon[22742]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:15:58.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:57 smithi059 ceph-mon[21819]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:15:58.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:57 smithi059 ceph-mon[26536]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:15:58.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:58 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:59.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:58 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:59.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:58 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:15:59.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:15:59 smithi064 ceph-mon[22742]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:59.984 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:15:59 smithi059 ceph-mon[26536]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:15:59.984 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:15:59 smithi059 ceph-mon[21819]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:01 smithi064 ceph-mon[22742]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:02.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:01 smithi059 ceph-mon[21819]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:02.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:01 smithi059 ceph-mon[26536]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:03.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:16:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:16:02] "GET /metrics HTTP/1.1" 200 38049 "" "Prometheus/2.33.4" 2024-04-26T16:16:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:03 smithi064 ceph-mon[22742]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:03 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:03.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:03 smithi059 ceph-mon[26536]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:03.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:03 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:03 smithi059 ceph-mon[21819]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:03 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:05.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:05 smithi064 ceph-mon[22742]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:06.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:05 smithi059 ceph-mon[21819]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:06.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:05 smithi059 ceph-mon[26536]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:07 smithi064 ceph-mon[22742]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:08.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:07 smithi059 ceph-mon[21819]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:08.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:07 smithi059 ceph-mon[26536]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:09.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:09 smithi064 ceph-mon[22742]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:10.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:09 smithi059 ceph-mon[26536]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:10.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:09 smithi059 ceph-mon[21819]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:11.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:11 smithi064 ceph-mon[22742]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:12.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:11 smithi059 ceph-mon[26536]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:12.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:11 smithi059 ceph-mon[21819]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:13.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:16:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:16:12] "GET /metrics HTTP/1.1" 200 38049 "" "Prometheus/2.33.4" 2024-04-26T16:16:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:13 smithi064 ceph-mon[22742]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:13 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:13.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:13 smithi059 ceph-mon[21819]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:13.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:13 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:13.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:13 smithi059 ceph-mon[26536]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:13.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:13 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:14 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:14 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:14 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:15 smithi064 ceph-mon[22742]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:15.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:15 smithi059 ceph-mon[26536]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:15 smithi059 ceph-mon[21819]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:17.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:17 smithi064 ceph-mon[22742]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:18.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:17 smithi059 ceph-mon[26536]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:18.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:17 smithi059 ceph-mon[21819]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:19.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:19 smithi064 ceph-mon[22742]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:20.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:19 smithi059 ceph-mon[21819]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:20.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:19 smithi059 ceph-mon[26536]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:21.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:21 smithi064 ceph-mon[22742]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:22.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:21 smithi059 ceph-mon[21819]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:22.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:21 smithi059 ceph-mon[26536]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:23.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:16:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:16:22] "GET /metrics HTTP/1.1" 200 38052 "" "Prometheus/2.33.4" 2024-04-26T16:16:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:23 smithi064 ceph-mon[22742]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:23 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:23.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:23 smithi059 ceph-mon[21819]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:23.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:23 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:23 smithi059 ceph-mon[26536]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:23 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:25 smithi064 ceph-mon[22742]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:26.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:25 smithi059 ceph-mon[21819]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:26.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:25 smithi059 ceph-mon[26536]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:27 smithi064 ceph-mon[22742]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:28.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:27 smithi059 ceph-mon[21819]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:28.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:27 smithi059 ceph-mon[26536]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:28 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:29.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:28 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:29.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:28 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:29.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:29 smithi064 ceph-mon[22742]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:29.984 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:29 smithi059 ceph-mon[21819]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:29.984 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:29 smithi059 ceph-mon[26536]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:31 smithi064 ceph-mon[22742]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:32.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:31 smithi059 ceph-mon[21819]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:32.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:31 smithi059 ceph-mon[26536]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:33.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:16:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:16:32] "GET /metrics HTTP/1.1" 200 38049 "" "Prometheus/2.33.4" 2024-04-26T16:16:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:33 smithi064 ceph-mon[22742]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:33 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:33.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:33 smithi059 ceph-mon[21819]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:33.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:33 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:33.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:33 smithi059 ceph-mon[26536]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:33.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:33 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:35 smithi064 ceph-mon[22742]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:36.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:35 smithi059 ceph-mon[21819]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:35 smithi059 ceph-mon[26536]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:37 smithi064 ceph-mon[22742]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:38.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:37 smithi059 ceph-mon[21819]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:38.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:37 smithi059 ceph-mon[26536]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:39 smithi064 ceph-mon[22742]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:40.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:39 smithi059 ceph-mon[21819]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:39 smithi059 ceph-mon[26536]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:41.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:41 smithi064 ceph-mon[22742]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:41 smithi059 ceph-mon[21819]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:42.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:41 smithi059 ceph-mon[26536]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:43.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:16:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:16:42] "GET /metrics HTTP/1.1" 200 38049 "" "Prometheus/2.33.4" 2024-04-26T16:16:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:43 smithi064 ceph-mon[22742]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:43 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:43.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:43 smithi059 ceph-mon[21819]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:43 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:43.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:43 smithi059 ceph-mon[26536]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:43.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:43 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:44 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:44.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:44 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:44 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:45.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:45 smithi064 ceph-mon[22742]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:45.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:45 smithi059 ceph-mon[21819]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:45.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:45 smithi059 ceph-mon[26536]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:16:47.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:47 smithi064 ceph-mon[22742]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:47.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:47 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:16:47.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:47 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:16:47.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:47 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:16:47.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:47 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:16:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:47 smithi059 ceph-mon[21819]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:47 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:16:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:47 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:16:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:47 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:16:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:47 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:16:48.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:47 smithi059 ceph-mon[26536]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:16:48.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:47 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:16:48.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:47 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:16:48.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:47 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:16:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:47 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:16:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:48 smithi064 ceph-mon[22742]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 951 B/s rd, 0 op/s 2024-04-26T16:16:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:48 smithi064 ceph-mon[22742]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:16:49.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:48 smithi059 ceph-mon[21819]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 951 B/s rd, 0 op/s 2024-04-26T16:16:49.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:48 smithi059 ceph-mon[21819]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:16:49.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:48 smithi059 ceph-mon[26536]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 951 B/s rd, 0 op/s 2024-04-26T16:16:49.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:48 smithi059 ceph-mon[26536]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:16:50.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:50 smithi064 ceph-mon[22742]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-04-26T16:16:51.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:50 smithi059 ceph-mon[21819]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-04-26T16:16:51.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:50 smithi059 ceph-mon[26536]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-04-26T16:16:52.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:52 smithi064 ceph-mon[22742]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:16:52.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:16:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:16:52] "GET /metrics HTTP/1.1" 200 38055 "" "Prometheus/2.33.4" 2024-04-26T16:16:53.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:52 smithi059 ceph-mon[21819]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:16:53.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:52 smithi059 ceph-mon[26536]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:16:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:53 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:54.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:53 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:54.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:53 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:16:54.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:54 smithi064 ceph-mon[22742]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-04-26T16:16:55.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:54 smithi059 ceph-mon[26536]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-04-26T16:16:55.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:54 smithi059 ceph-mon[21819]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-04-26T16:16:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:56 smithi064 ceph-mon[22742]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:16:57.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:56 smithi059 ceph-mon[21819]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:16:57.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:56 smithi059 ceph-mon[26536]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:16:58.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:58 smithi064 ceph-mon[22742]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:16:58.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:16:58 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:59.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:58 smithi059 ceph-mon[21819]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:16:59.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:16:58 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:16:59.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:58 smithi059 ceph-mon[26536]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:16:59.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:16:58 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:00.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:00 smithi064 ceph-mon[22742]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:01.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:00 smithi059 ceph-mon[21819]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:01.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:00 smithi059 ceph-mon[26536]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:02.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:02 smithi064 ceph-mon[22742]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:02.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:17:02] "GET /metrics HTTP/1.1" 200 38057 "" "Prometheus/2.33.4" 2024-04-26T16:17:03.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:02 smithi059 ceph-mon[21819]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:02 smithi059 ceph-mon[26536]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:03.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:03 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:04.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:03 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:04.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:03 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:04 smithi064 ceph-mon[22742]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:04 smithi059 ceph-mon[21819]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:05.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:04 smithi059 ceph-mon[26536]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:06.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:06 smithi064 ceph-mon[22742]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:07.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:06 smithi059 ceph-mon[21819]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:07.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:06 smithi059 ceph-mon[26536]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:09.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:08 smithi059 ceph-mon[21819]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:09.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:08 smithi059 ceph-mon[26536]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:09.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:08 smithi064 ceph-mon[22742]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:11.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:10 smithi059 ceph-mon[21819]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:11.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:10 smithi059 ceph-mon[26536]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:11.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:10 smithi064 ceph-mon[22742]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:13.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:12 smithi059 ceph-mon[21819]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:13.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:12 smithi059 ceph-mon[26536]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:13.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:12 smithi064 ceph-mon[22742]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:13.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:17:12] "GET /metrics HTTP/1.1" 200 38057 "" "Prometheus/2.33.4" 2024-04-26T16:17:14.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:13 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:14.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:13 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:14.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:13 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:14.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:13 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:14.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:13 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:14.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:13 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:15.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:14 smithi059 ceph-mon[21819]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:15.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:14 smithi059 ceph-mon[26536]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:15.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:14 smithi064 ceph-mon[22742]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:16 smithi059 ceph-mon[21819]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:17.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:16 smithi059 ceph-mon[26536]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:17.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:16 smithi064 ceph-mon[22742]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:19.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:18 smithi059 ceph-mon[21819]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:19.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:18 smithi059 ceph-mon[26536]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:19.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:18 smithi064 ceph-mon[22742]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:21.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:20 smithi059 ceph-mon[21819]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:21.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:20 smithi059 ceph-mon[26536]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:21.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:20 smithi064 ceph-mon[22742]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:23.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:22 smithi059 ceph-mon[21819]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:23.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:22 smithi059 ceph-mon[26536]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:23.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:22 smithi064 ceph-mon[22742]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:23.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:17:22] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.33.4" 2024-04-26T16:17:24.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:23 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:24.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:23 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:24.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:23 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:25.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:24 smithi059 ceph-mon[21819]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:24 smithi059 ceph-mon[26536]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:25.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:24 smithi064 ceph-mon[22742]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:27.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:26 smithi059 ceph-mon[21819]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:27.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:26 smithi059 ceph-mon[26536]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:27.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:26 smithi064 ceph-mon[22742]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:29.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:28 smithi059 ceph-mon[21819]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:29.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:28 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:28 smithi059 ceph-mon[26536]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:28 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:29.199 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:28 smithi064 ceph-mon[22742]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:29.199 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:28 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:30.885 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:30 smithi059 ceph-mon[21819]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:30.885 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:30 smithi059 ceph-mon[26536]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:31.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:30 smithi064 ceph-mon[22742]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:31.471 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:17:32.791 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:32 smithi059 ceph-mon[21819]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:32.791 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:32 smithi059 ceph-mon[26536]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:33.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:32 smithi064 ceph-mon[22742]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:33.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:17:32] "GET /metrics HTTP/1.1" 200 38057 "" "Prometheus/2.33.4" 2024-04-26T16:17:33.498 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:17:33.498 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (12m) 2m ago 20m 24.2M - 0.23.0 ba2b418f427c ca6d4a23fcc2 2024-04-26T16:17:33.498 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (12m) 2m ago 20m 55.2M - 8.3.5 dad864ee21e9 0d16158d9791 2024-04-26T16:17:33.498 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 error 2m ago 19m 54.9M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283 running (15m) 2m ago 24m 505M - 19.0.0-3305-ga430f9ed 031ba2747cfd 03edb3b65e45 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (9m) 2m ago 26m 87.7M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (26m) 2m ago 26m 69.3M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (25m) 2m ago 25m 55.7M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (25m) 2m ago 25m 55.9M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (12m) 2m ago 21m 21.3M - 1.3.1 1dbe0e931976 9b14cfbf9c80 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (12m) 2m ago 20m 17.7M - 1.3.1 1dbe0e931976 fb96f1dc25f3 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (24m) 2m ago 24m 53.3M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (24m) 2m ago 24m 53.1M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (23m) 2m ago 23m 51.8M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (23m) 2m ago 23m 52.4M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (22m) 2m ago 22m 52.7M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:17:33.499 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (22m) 2m ago 22m 53.4M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:17:33.500 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (22m) 2m ago 22m 50.6M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:17:33.500 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (21m) 2m ago 21m 53.1M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:17:33.500 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (2m) 2m ago 21m 44.5M - 2.33.4 514e6a882f6e 2614c60ef06f 2024-04-26T16:17:33.500 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (19m) 2m ago 19m 74.2M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:17:33.500 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (19m) 2m ago 19m 74.7M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:17:34.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:33 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:34.047 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:33 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:34.088 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:33 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:34.211 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ls' 2024-04-26T16:17:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:34 smithi059 ceph-mon[21819]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:34 smithi059 ceph-mon[21819]: from='client.15348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:34 smithi059 ceph-mon[26536]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:34 smithi059 ceph-mon[26536]: from='client.15348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:35.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:34 smithi064 ceph-mon[22742]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:35.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:34 smithi064 ceph-mon[22742]: from='client.15348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:36.193 INFO:teuthology.orchestra.run.smithi059.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-26T16:17:36.194 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi059=a;count:1 2024-04-26T16:17:36.194 INFO:teuthology.orchestra.run.smithi059.stdout:grafana ?:3000 1/1 2m ago 21m smithi064=a;count:1 2024-04-26T16:17:36.194 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo ?:5000 0/1 3m ago 19m count:1 2024-04-26T16:17:36.194 INFO:teuthology.orchestra.run.smithi059.stdout:mgr 2/2 3m ago 24m smithi059=y;smithi064=x;count:2 2024-04-26T16:17:36.194 INFO:teuthology.orchestra.run.smithi059.stdout:mon 3/3 3m ago 25m smithi059:172.21.15.59=a;smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c;smithi064:172.21.15.64=b;count:3 2024-04-26T16:17:36.194 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi059=a;smithi064=b;count:2 2024-04-26T16:17:36.194 INFO:teuthology.orchestra.run.smithi059.stdout:osd 8 3m ago - 2024-04-26T16:17:36.194 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus ?:9095 1/1 2m ago 21m smithi064=a;count:1 2024-04-26T16:17:36.194 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-04-26T16:17:36.774 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions' 2024-04-26T16:17:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:36 smithi059 ceph-mon[26536]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:37.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:36 smithi059 ceph-mon[21819]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:37.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:36 smithi064 ceph-mon[22742]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:38.069 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:37 smithi059 ceph-mon[21819]: from='client.25267 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:38.069 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:37 smithi059 ceph-mon[26536]: from='client.25267 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:38.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:37 smithi064 ceph-mon[22742]: from='client.25267 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:17:38.877 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:17:38.878 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:17:38.878 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:17:38.878 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:17:38.878 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:17:38.878 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T16:17:38.878 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:17:38.878 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:17:38.878 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:17:39.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:38 smithi059 ceph-mon[21819]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:38 smithi059 ceph-mon[26536]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:39.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:38 smithi064 ceph-mon[22742]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:39.435 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-04-26T16:17:40.079 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:39 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3025988850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:17:40.079 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:39 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3025988850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:17:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:39 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3025988850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:17:40.968 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:40 smithi059 ceph-mon[26536]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:40.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:40 smithi059 ceph-mon[21819]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:41.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:40 smithi064 ceph-mon[22742]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:43.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:42 smithi059 ceph-mon[26536]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:42 smithi059 ceph-mon[26536]: from='client.15366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:43.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:42 smithi059 ceph-mon[21819]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:43.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:42 smithi059 ceph-mon[21819]: from='client.15366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:43.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:42 smithi064 ceph-mon[22742]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:43.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:42 smithi064 ceph-mon[22742]: from='client.15366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:43.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:42 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:17:42] "GET /metrics HTTP/1.1" 200 38057 "" "Prometheus/2.33.4" 2024-04-26T16:17:43.431 INFO:teuthology.orchestra.run.smithi059.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:17:44.040 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:44.040 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:17:44.040 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:17:44.040 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:17:44.040 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:17:44.041 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:17:44.041 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:44.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:44.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:17:44.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:17:44.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:17:44.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:17:44.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:17:44.041 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:43 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:44.064 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-26T16:17:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:43 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:43 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:17:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:43 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:17:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:43 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:17:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:43 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:17:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:43 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:17:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:43 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:44 smithi059 ceph-mon[26536]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:44 smithi059 ceph-mon[26536]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:17:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:44 smithi059 ceph-mon[26536]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-04-26T16:17:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:44 smithi059 ceph-mon[26536]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-26T16:17:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:44 smithi059 ceph-mon[26536]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:17:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:44 smithi059 ceph-mon[21819]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:44 smithi059 ceph-mon[21819]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:17:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:44 smithi059 ceph-mon[21819]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-04-26T16:17:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:44 smithi059 ceph-mon[21819]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-26T16:17:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:44 smithi059 ceph-mon[21819]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:17:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:44 smithi064 ceph-mon[22742]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:17:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:44 smithi064 ceph-mon[22742]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:17:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:44 smithi064 ceph-mon[22742]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-04-26T16:17:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:44 smithi064 ceph-mon[22742]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-26T16:17:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:44 smithi064 ceph-mon[22742]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:17:46.139 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:17:46.850 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:17:46.850 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (12m) 3m ago 20m 24.2M - 0.23.0 ba2b418f427c ca6d4a23fcc2 2024-04-26T16:17:46.850 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (12m) 3m ago 20m 55.2M - 8.3.5 dad864ee21e9 0d16158d9791 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 error 3m ago 19m 54.9M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283 running (15m) 3m ago 24m 505M - 19.0.0-3305-ga430f9ed 031ba2747cfd 03edb3b65e45 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (9m) 3m ago 26m 87.7M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (26m) 3m ago 26m 69.3M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (25m) 3m ago 25m 55.7M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (25m) 3m ago 25m 55.9M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (12m) 3m ago 21m 21.3M - 1.3.1 1dbe0e931976 9b14cfbf9c80 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (12m) 3m ago 21m 17.7M - 1.3.1 1dbe0e931976 fb96f1dc25f3 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (24m) 3m ago 24m 53.3M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (24m) 3m ago 24m 53.1M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (23m) 3m ago 23m 51.8M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (23m) 3m ago 23m 52.4M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (23m) 3m ago 23m 52.7M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:17:46.851 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (22m) 3m ago 22m 53.4M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:17:46.852 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (22m) 3m ago 22m 50.6M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:17:46.852 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (22m) 3m ago 22m 53.1M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:17:46.852 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (3m) 3m ago 21m 44.5M - 2.33.4 514e6a882f6e 2614c60ef06f 2024-04-26T16:17:46.852 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (19m) 3m ago 19m 74.2M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:17:46.852 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (19m) 3m ago 19m 74.7M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:17:46.901 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:46 smithi059 ceph-mon[21819]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:46.901 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:46 smithi059 ceph-mon[26536]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:47.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:46 smithi064 ceph-mon[22742]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:17:47.281 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:17:47.281 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:17:47.281 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:17:47.281 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:17:47.281 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:17:47.281 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:17:47.282 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:17:47.637 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:17:47.638 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:17:47.638 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:17:47.638 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-26T16:17:47.638 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [ 2024-04-26T16:17:47.638 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr" 2024-04-26T16:17:47.639 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T16:17:47.639 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "2/2 daemons upgraded", 2024-04-26T16:17:47.639 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 image", 2024-04-26T16:17:47.639 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:17:47.639 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:17:47.967 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[26536]: from='client.25285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:47.967 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[26536]: from='client.25291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:47.967 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[26536]: from='client.25297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:47.967 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:17:47.967 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:17:47.967 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[26536]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T16:17:47.967 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T16:17:47.967 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[26536]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T16:17:47.967 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1727356751' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:17:47.967 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[26536]: Standby manager daemon x started 2024-04-26T16:17:47.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[21819]: from='client.25285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:47.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[21819]: from='client.25291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:47.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[21819]: from='client.25297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:47.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:17:47.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:17:47.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[21819]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T16:17:47.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T16:17:47.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[21819]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T16:17:47.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1727356751' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:17:47.968 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-mon[21819]: Standby manager daemon x started 2024-04-26T16:17:48.104 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:47.956+0000 7f15bd7fa700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T16:17:48.104 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ignoring --setuser ceph since I am not root 2024-04-26T16:17:48.104 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: ignoring --setgroup ceph since I am not root 2024-04-26T16:17:48.104 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mgr[42967]: -- 172.21.15.64:0/3094598576 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c87dd585a0 con 0x55c87dd2d000 2024-04-26T16:17:48.104 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-mon[22742]: from='client.25285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:48.104 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-mon[22742]: from='client.25291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:48.104 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-mon[22742]: from='client.25297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:48.104 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' 2024-04-26T16:17:48.105 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:17:48.105 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-mon[22742]: from='mgr.35024 172.21.15.64:0/3659022892' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T16:17:48.105 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T16:17:48.105 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-mon[22742]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T16:17:48.105 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1727356751' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:17:48.105 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:47 smithi064 ceph-mon[22742]: Standby manager daemon x started 2024-04-26T16:17:48.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:17:47] ENGINE Bus STOPPING 2024-04-26T16:17:48.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:17:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:17:48.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:17:47] ENGINE Bus STOPPED 2024-04-26T16:17:48.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:17:48] ENGINE Bus STARTING 2024-04-26T16:17:48.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:17:48] ENGINE Serving on http://:::9283 2024-04-26T16:17:48.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:17:48] ENGINE Bus STARTED 2024-04-26T16:17:48.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:48.143+0000 7f71b1693200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T16:17:48.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:48.338+0000 7f71b1693200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:17:48.777 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:48.449+0000 7f71b1693200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T16:17:49.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:17:49.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:17:49.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T16:17:49.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: Failing over to other MGR 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='client.15396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.35024 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: mgrmap e38: y(active, starting, since 0.996904s), standbys: x 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:17:49.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: Manager daemon y is now available 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:17:49.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: Failing over to other MGR 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='client.15396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.35024 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: mgrmap e38: y(active, starting, since 0.996904s), standbys: x 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:17:49.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: Manager daemon y is now available 2024-04-26T16:17:49.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:17:49.088 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:49.088 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:17:49.088 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:17:49.088 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:17:49.088 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:48 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:17:49.143 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:48.781+0000 7f71b1693200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T16:17:49.143 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:48.881+0000 7f71b1693200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T16:17:49.144 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:17:49.144 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:17:49.144 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T16:17:49.144 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T16:17:49.144 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: Failing over to other MGR 2024-04-26T16:17:49.144 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='client.15396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.35024 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: mgrmap e38: y(active, starting, since 0.996904s), standbys: x 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:17:49.145 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: Manager daemon y is now available 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:17:49.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:48 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:17:50.137 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:49 smithi064 ceph-mon[22742]: [26/Apr/2024:16:17:48] ENGINE Bus STARTING 2024-04-26T16:17:50.137 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:49 smithi064 ceph-mon[22742]: [26/Apr/2024:16:17:48] ENGINE Serving on http://172.21.15.59:8765 2024-04-26T16:17:50.137 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:49 smithi064 ceph-mon[22742]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-26T16:17:50.137 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:49 smithi064 ceph-mon[22742]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:50.137 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:49 smithi064 ceph-mon[22742]: [26/Apr/2024:16:17:48] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T16:17:50.137 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:49 smithi064 ceph-mon[22742]: [26/Apr/2024:16:17:48] ENGINE Bus STARTED 2024-04-26T16:17:50.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[21819]: [26/Apr/2024:16:17:48] ENGINE Bus STARTING 2024-04-26T16:17:50.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[21819]: [26/Apr/2024:16:17:48] ENGINE Serving on http://172.21.15.59:8765 2024-04-26T16:17:50.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[21819]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-26T16:17:50.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[21819]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:50.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[21819]: [26/Apr/2024:16:17:48] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T16:17:50.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[21819]: [26/Apr/2024:16:17:48] ENGINE Bus STARTED 2024-04-26T16:17:50.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[26536]: [26/Apr/2024:16:17:48] ENGINE Bus STARTING 2024-04-26T16:17:50.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[26536]: [26/Apr/2024:16:17:48] ENGINE Serving on http://172.21.15.59:8765 2024-04-26T16:17:50.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[26536]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-26T16:17:50.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[26536]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:50.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[26536]: [26/Apr/2024:16:17:48] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T16:17:50.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:49 smithi059 ceph-mon[26536]: [26/Apr/2024:16:17:48] ENGINE Bus STARTED 2024-04-26T16:17:50.436 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:50.187+0000 7f71b1693200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T16:17:50.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:50.367+0000 7f71b1693200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T16:17:50.884 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:50.567+0000 7f71b1693200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T16:17:50.884 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:50.884+0000 7f71b1693200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T16:17:51.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:50 smithi064 ceph-mon[22742]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T16:17:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:50 smithi064 ceph-mon[22742]: Cluster is now healthy 2024-04-26T16:17:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:50 smithi064 ceph-mon[22742]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:51.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:50.991+0000 7f71b1693200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T16:17:51.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:50 smithi059 ceph-mon[21819]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T16:17:51.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:50 smithi059 ceph-mon[21819]: Cluster is now healthy 2024-04-26T16:17:51.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:50 smithi059 ceph-mon[21819]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:51.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:50 smithi059 ceph-mon[26536]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T16:17:51.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:50 smithi059 ceph-mon[26536]: Cluster is now healthy 2024-04-26T16:17:51.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:50 smithi059 ceph-mon[26536]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:51.585 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:51.205+0000 7f71b1693200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T16:17:51.585 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:51.423+0000 7f71b1693200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:17:52.309 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:51 smithi064 ceph-mon[22742]: mgrmap e40: y(active, since 4s), standbys: x 2024-04-26T16:17:52.309 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:52.063+0000 7f71b1693200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T16:17:52.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:51 smithi059 ceph-mon[26536]: mgrmap e40: y(active, since 4s), standbys: x 2024-04-26T16:17:52.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:51 smithi059 ceph-mon[21819]: mgrmap e40: y(active, since 4s), standbys: x 2024-04-26T16:17:52.589 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:52.309+0000 7f71b1693200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-04-26T16:17:52.872 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:52.770+0000 7f71b1693200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T16:17:53.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:52.872+0000 7f71b1693200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T16:17:53.680 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:53.288+0000 7f71b1693200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T16:17:53.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:53 smithi064 ceph-mon[22742]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:53.680 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:53 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:53 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:53 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.681 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:53 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:53 smithi059 ceph-mon[26536]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:53 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:53 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:53 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:53 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:53 smithi059 ceph-mon[21819]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:53 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:53 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:53 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:53 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:53.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:53.680+0000 7f71b1693200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T16:17:53.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:53.782+0000 7f71b1693200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T16:17:54.383 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:54.052+0000 7f71b1693200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T16:17:54.383 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:54.198+0000 7f71b1693200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T16:17:54.383 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:54.301+0000 7f71b1693200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T16:17:54.582 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=warn ts=2024-04-26T16:17:54.246Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:17:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:17:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:17:54.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:54.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:17:54.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:54 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:17:54.641 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:17:54.641 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.641 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.641 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.641 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:54.641 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:17:54.641 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:17:54.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:54.640+0000 7f71b1693200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T16:17:54.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:54 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:54.740+0000 7f71b1693200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T16:17:55.640 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:55.397+0000 7f71b1693200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T16:17:55.640 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-mon[22742]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:55.640 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:17:55.640 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:17:55.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:55 smithi059 ceph-mon[21819]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:55 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:17:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:55 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:17:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:55 smithi059 ceph-mon[26536]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:17:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:55 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:17:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:55 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:17:55.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:55.639+0000 7f71b1693200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T16:17:55.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:17:55] ENGINE Bus STARTING 2024-04-26T16:17:55.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: CherryPy Checker: 2024-04-26T16:17:55.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: The Application mounted at '' has an empty config. 2024-04-26T16:17:55.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: 2024-04-26T16:17:55.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:17:55] ENGINE Serving on http://:::9283 2024-04-26T16:17:55.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:17:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[42963]: [26/Apr/2024:16:17:55] ENGINE Bus STARTED 2024-04-26T16:17:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:17:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:17:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:17:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:17:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:17:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:17:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: Standby manager daemon x restarted 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: Standby manager daemon x started 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:17:56.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: Standby manager daemon x restarted 2024-04-26T16:17:56.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: Standby manager daemon x started 2024-04-26T16:17:56.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:17:56.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:17:56.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:17:56.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:17:56.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:17:56.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:17:56.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:17:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:17:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:17:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:17:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:17:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:17:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:17:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: Standby manager daemon x restarted 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: Standby manager daemon x started 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/2132635656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:17:56.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:56 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:17:57.303 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:17:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:17:56.901Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2024-04-26T16:17:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-mon[21819]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 18 op/s 2024-04-26T16:17:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:17:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:17:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-mon[21819]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:17:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-mon[21819]: mgrmap e41: y(active, since 9s), standbys: x 2024-04-26T16:17:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-mon[26536]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 18 op/s 2024-04-26T16:17:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:17:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:17:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-mon[26536]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:17:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-mon[26536]: mgrmap e41: y(active, since 9s), standbys: x 2024-04-26T16:17:57.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:17:57.468+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:17:57.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:17:57.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:17:57.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:17:57.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:17:57.589 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:17:57.589 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:57.589 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:17:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:57 smithi064 ceph-mon[22742]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 18 op/s 2024-04-26T16:17:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:57 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:17:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:57 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:17:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:57 smithi064 ceph-mon[22742]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:17:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:57 smithi064 ceph-mon[22742]: mgrmap e41: y(active, since 9s), standbys: x 2024-04-26T16:17:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:58 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:17:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:17:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:17:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:17:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:17:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:17:58.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:17:58.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:17:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:17:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:17:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:17:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:17:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:17:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:17:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:17:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:58 smithi064 ceph-mon[22742]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:17:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:58 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:17:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:17:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:17:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:17:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:17:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:17:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:17:58.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:17:58.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:17:58.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:58 smithi059 ceph-mon[21819]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:58 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:17:58.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:17:58.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:17:58.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:17:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:58 smithi059 ceph-mon[26536]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T16:17:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:59 smithi064 ceph-mon[22742]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T16:17:59.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:17:59 smithi064 ceph-mon[22742]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:17:59.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:59 smithi059 ceph-mon[21819]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T16:17:59.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:17:59 smithi059 ceph-mon[21819]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:17:59.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:59 smithi059 ceph-mon[26536]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T16:17:59.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:17:59 smithi059 ceph-mon[26536]: Reconfiguring daemon prometheus.a on smithi064 2024-04-26T16:18:01.356 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 systemd[1]: Stopping Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:18:01.356 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 bash[57573]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-26T16:18:01.648 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-mon[22742]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.356Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-26T16:18:01.649 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[54778]: ts=2024-04-26T16:18:01.357Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-26T16:18:01.650 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 podman[57586]: 2024-04-26 16:18:01.367164114 +0000 UTC m=+0.114808472 container died 2614c60ef06fa639bb199e2cbd3b28f4af74d935b9c89e34dae28538afbf4896 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:01.654 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:01 smithi059 ceph-mon[26536]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T16:18:01.654 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:01 smithi059 ceph-mon[21819]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T16:18:01.937 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 podman[57586]: 2024-04-26 16:18:01.756786324 +0000 UTC m=+0.504430671 container cleanup 2614c60ef06fa639bb199e2cbd3b28f4af74d935b9c89e34dae28538afbf4896 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:01.938 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 bash[57586]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T16:18:01.938 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:01 smithi064 bash[57612]: Error: no container with name or ID "ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-26T16:18:02.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 podman[57598]: 2024-04-26 16:18:02.167055296 +0000 UTC m=+0.800765335 container remove 2614c60ef06fa639bb199e2cbd3b28f4af74d935b9c89e34dae28538afbf4896 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:02.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T16:18:02.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 systemd[1]: Stopped Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:18:02.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.439s CPU time. 2024-04-26T16:18:02.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 systemd[1]: Starting Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:18:02.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 podman[57680]: 2024-04-26 16:18:02.306624611 +0000 UTC m=+0.019353176 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-26T16:18:02.437 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 podman[57680]: 2024-04-26 16:18:02.409819366 +0000 UTC m=+0.122547920 container create eb5926cbd0ddee1b90d203d164316e6fcfc0207d95c0b834c2d9b7dbb523df90 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 podman[57680]: 2024-04-26 16:18:02.628538107 +0000 UTC m=+0.341266663 container init eb5926cbd0ddee1b90d203d164316e6fcfc0207d95c0b834c2d9b7dbb523df90 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 podman[57680]: 2024-04-26 16:18:02.631191367 +0000 UTC m=+0.343919923 container start eb5926cbd0ddee1b90d203d164316e6fcfc0207d95c0b834c2d9b7dbb523df90 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.655Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.655Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.655Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.655Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi064 (none))" 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.655Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.655Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.657Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.657Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-26T16:18:02.866 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.659Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-26T16:18:02.867 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.661Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T16:18:02.867 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.662Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.4191ms 2024-04-26T16:18:02.867 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.662Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T16:18:02.867 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 bash[57680]: eb5926cbd0ddee1b90d203d164316e6fcfc0207d95c0b834c2d9b7dbb523df90 2024-04-26T16:18:02.867 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 systemd[1]: Started Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:18:02.867 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:02.866Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-04-26T16:18:03.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:18:02] ENGINE Bus STOPPING 2024-04-26T16:18:03.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:18:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:18:03.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:18:02] ENGINE Bus STOPPED 2024-04-26T16:18:03.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:18:02] ENGINE Bus STARTING 2024-04-26T16:18:03.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:18:02] ENGINE Serving on http://:::9283 2024-04-26T16:18:03.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:18:02] ENGINE Bus STARTED 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.120Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.339Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.342Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.349Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.349Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.349Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=54.472µs wal_replay_duration=687.028398ms total_replay_duration=688.521659ms 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.354Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.354Z caller=main.go:947 level=info msg="TSDB started" 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.354Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.375Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.16349ms db_storage=797ns remote_storage=1.248µs web_handler=318ns query_engine=720ns scrape=377.054µs scrape_sd=95.063µs notify=11.975µs notify_sd=9.384µs rules=20.352577ms 2024-04-26T16:18:03.375 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:18:03.375Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-26T16:18:03.648 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-mon[22742]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:18:03.648 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:03.648 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:03.648 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:18:03.648 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:18:03.648 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:03.648 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:03.648 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[21819]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:18:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:18:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:18:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:03.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[26536]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:18:03.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:03.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:03.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:18:03.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:18:03.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:03.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:03.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:04 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:18:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:04 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:18:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:04 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:04 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:18:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:04 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:18:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:04 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:04 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:18:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:04 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi064.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T16:18:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:04 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:05 smithi064 ceph-mon[22742]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:18:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:05 smithi064 ceph-mon[22742]: Upgrade: Updating mgr.x 2024-04-26T16:18:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:05 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:18:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:05 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:18:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:05 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:18:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:05 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:05 smithi064 ceph-mon[22742]: Deploying daemon mgr.x on smithi064 2024-04-26T16:18:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[21819]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:18:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[21819]: Upgrade: Updating mgr.x 2024-04-26T16:18:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:18:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:18:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:18:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[21819]: Deploying daemon mgr.x on smithi064 2024-04-26T16:18:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[26536]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:18:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[26536]: Upgrade: Updating mgr.x 2024-04-26T16:18:05.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:18:05.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:18:05.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:18:05.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:05.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:05 smithi059 ceph-mon[26536]: Deploying daemon mgr.x on smithi064 2024-04-26T16:18:06.561 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:06 smithi064 systemd[1]: Stopping Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:18:06.847 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:06 smithi064 podman[58030]: 2024-04-26 16:18:06.561390754 +0000 UTC m=+0.144798782 container died 03edb3b65e457699ad80c33ae495a553ed58d5bdb46cf7c44d6150fdc4e9b623 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:18:07.172 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:06 smithi064 podman[58030]: 2024-04-26 16:18:06.922089749 +0000 UTC m=+0.505497760 container cleanup 03edb3b65e457699ad80c33ae495a553ed58d5bdb46cf7c44d6150fdc4e9b623 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T16:18:07.172 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:06 smithi064 bash[58030]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x 2024-04-26T16:18:07.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:07 smithi064 ceph-mon[22742]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 0 B/s wr, 134 op/s 2024-04-26T16:18:07.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:07 smithi064 podman[58044]: 2024-04-26 16:18:07.173208684 +0000 UTC m=+0.613822898 container remove 03edb3b65e457699ad80c33ae495a553ed58d5bdb46cf7c44d6150fdc4e9b623 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5) 2024-04-26T16:18:07.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:07 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T16:18:07.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:07 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T16:18:07.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:07 smithi064 systemd[1]: Stopped Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:18:07.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:07 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service: Consumed 1min 3.005s CPU time. 2024-04-26T16:18:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:07 smithi059 ceph-mon[21819]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 0 B/s wr, 134 op/s 2024-04-26T16:18:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:07 smithi059 ceph-mon[26536]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 0 B/s wr, 134 op/s 2024-04-26T16:18:07.888 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:07 smithi064 systemd[1]: Starting Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:18:07.888 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:07 smithi064 podman[58175]: 2024-04-26 16:18:07.615522669 +0000 UTC m=+0.018597381 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:18:07.888 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:07 smithi064 podman[58175]: 2024-04-26 16:18:07.724268065 +0000 UTC m=+0.127342822 container create b30a628a06dcefc2fa466d0ff187d22a26f866f8a18569f399d167b622e80368 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-04-26T16:18:08.350 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:07 smithi064 podman[58175]: 2024-04-26 16:18:07.993802391 +0000 UTC m=+0.396877104 container init b30a628a06dcefc2fa466d0ff187d22a26f866f8a18569f399d167b622e80368 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:18:08.350 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:07 smithi064 podman[58175]: 2024-04-26 16:18:07.996471913 +0000 UTC m=+0.399546628 container start b30a628a06dcefc2fa466d0ff187d22a26f866f8a18569f399d167b622e80368 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-26T16:18:08.350 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:08 smithi064 ceph-mgr[58194]: -- 172.21.15.64:0/485343842 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d1d9c105a0 con 0x55d1d9be4800 2024-04-26T16:18:08.350 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:08 smithi064 bash[58175]: b30a628a06dcefc2fa466d0ff187d22a26f866f8a18569f399d167b622e80368 2024-04-26T16:18:08.350 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:08 smithi064 systemd[1]: Started Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:18:08.350 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:08.153+0000 7fa32883e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T16:18:08.350 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:08.349+0000 7fa32883e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:18:08.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:08.459+0000 7fa32883e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T16:18:09.138 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:08.807+0000 7fa32883e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T16:18:09.139 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:08.895+0000 7fa32883e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T16:18:09.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:09 smithi064 ceph-mon[22742]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:18:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:09 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:09 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:09.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:09 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:09.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:09 smithi059 ceph-mon[26536]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:18:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:09 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:09 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:09 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:09 smithi059 ceph-mon[21819]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-26T16:18:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:09 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:09 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:09 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:10.392 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:10.194+0000 7fa32883e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T16:18:10.392 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:10.392+0000 7fa32883e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T16:18:10.680 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:10.582+0000 7fa32883e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T16:18:10.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:10.893+0000 7fa32883e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T16:18:11.423 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:11 smithi064 ceph-mon[22742]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:18:11.424 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:10.991+0000 7fa32883e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T16:18:11.424 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:11.219+0000 7fa32883e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T16:18:11.424 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:11.423+0000 7fa32883e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:18:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:11 smithi059 ceph-mon[26536]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:18:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:11 smithi059 ceph-mon[21819]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-26T16:18:12.403 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:12.069+0000 7fa32883e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T16:18:12.403 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:12.310+0000 7fa32883e200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-04-26T16:18:13.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:12.773+0000 7fa32883e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T16:18:13.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:12.878+0000 7fa32883e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T16:18:13.674 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:13.285+0000 7fa32883e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T16:18:13.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:13 smithi064 ceph-mon[22742]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-26T16:18:13.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:13 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:13.675 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:13 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:13.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:13 smithi059 ceph-mon[21819]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-26T16:18:13.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:13.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:13.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:13 smithi059 ceph-mon[26536]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-26T16:18:13.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:13.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:13.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:13.674+0000 7fa32883e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T16:18:13.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:13.775+0000 7fa32883e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T16:18:14.300 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:14.048+0000 7fa32883e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T16:18:14.300 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:14.193+0000 7fa32883e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T16:18:14.301 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:14.300+0000 7fa32883e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T16:18:14.391 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:18:14] "GET /metrics HTTP/1.1" 200 38057 "" "Prometheus/2.33.4" 2024-04-26T16:18:14.615 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:14 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:14.615 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:14 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:14.615 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:14 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:14.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:14 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:14 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:14 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:14 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:14 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:14 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:14.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:14.657+0000 7fa32883e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T16:18:14.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:14.759+0000 7fa32883e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T16:18:15.518 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:15 smithi064 ceph-mon[22742]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-26T16:18:15.518 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:15.411+0000 7fa32883e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T16:18:15.782 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:15.651+0000 7fa32883e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T16:18:15.782 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: [26/Apr/2024:16:18:15] ENGINE Bus STARTING 2024-04-26T16:18:15.782 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: CherryPy Checker: 2024-04-26T16:18:15.782 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: The Application mounted at '' has an empty config. 2024-04-26T16:18:15.782 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:18:15.782 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: [26/Apr/2024:16:18:15] ENGINE Serving on http://:::9283 2024-04-26T16:18:15.782 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:18:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: [26/Apr/2024:16:18:15] ENGINE Bus STARTED 2024-04-26T16:18:15.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:15 smithi059 ceph-mon[26536]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-26T16:18:15.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:15 smithi059 ceph-mon[21819]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-26T16:18:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:16 smithi064 ceph-mon[22742]: Standby manager daemon x restarted 2024-04-26T16:18:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:16 smithi064 ceph-mon[22742]: Standby manager daemon x started 2024-04-26T16:18:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:16 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:18:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:16 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:18:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:16 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:18:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:16 smithi064 ceph-mon[22742]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:18:16.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[26536]: Standby manager daemon x restarted 2024-04-26T16:18:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[26536]: Standby manager daemon x started 2024-04-26T16:18:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:18:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:18:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:18:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:18:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[21819]: Standby manager daemon x restarted 2024-04-26T16:18:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[21819]: Standby manager daemon x started 2024-04-26T16:18:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:18:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:18:16.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:18:16.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:16 smithi059 ceph-mon[21819]: from='mgr.? 172.21.15.64:0/4092759782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:18:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:17 smithi064 ceph-mon[22742]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-26T16:18:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:17 smithi064 ceph-mon[22742]: mgrmap e42: y(active, since 29s), standbys: x 2024-04-26T16:18:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:17 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:17 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:17 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:17 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:18:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:17 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:17 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:17 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:17 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:17.715 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[21819]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-26T16:18:17.715 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[21819]: mgrmap e42: y(active, since 29s), standbys: x 2024-04-26T16:18:17.715 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:17.715 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:17.715 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:17.715 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:18:17.715 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:17.716 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:17.716 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:17.716 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:17.716 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[26536]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-26T16:18:17.716 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[26536]: mgrmap e42: y(active, since 29s), standbys: x 2024-04-26T16:18:17.716 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:17.716 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:17.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:17.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:18:17.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:17.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:17.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:17.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:17 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:18.022 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:18:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:18 smithi064 ceph-mon[22742]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 0 op/s 2024-04-26T16:18:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:18 smithi064 ceph-mon[22742]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:18 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:18:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:18 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:18:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:18 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:18.722 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:18 smithi059 ceph-mon[26536]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 0 op/s 2024-04-26T16:18:18.722 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:18 smithi059 ceph-mon[26536]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:18.722 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:18 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:18:18.723 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:18 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:18:18.723 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:18 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:18.723 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:18 smithi059 ceph-mon[21819]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 0 op/s 2024-04-26T16:18:18.723 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:18 smithi059 ceph-mon[21819]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:18.723 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:18 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:18:18.723 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:18 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:18:18.723 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:18 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:18.724 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:18:18.724 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (13m) 26s ago 21m 23.8M - 0.23.0 ba2b418f427c ca6d4a23fcc2 2024-04-26T16:18:18.724 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (13m) 6s ago 20m 55.8M - 8.3.5 dad864ee21e9 0d16158d9791 2024-04-26T16:18:18.724 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 error 26s ago 20m 59.0M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:18:18.724 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (11s) 6s ago 25m 65.9M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:18:18.724 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (10m) 26s ago 27m 489M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:18:18.724 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (27m) 26s ago 27m 77.0M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:18:18.724 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (25m) 6s ago 25m 60.8M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (25m) 26s ago 25m 57.4M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (13m) 26s ago 21m 19.1M - 1.3.1 1dbe0e931976 9b14cfbf9c80 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (13m) 6s ago 21m 17.5M - 1.3.1 1dbe0e931976 fb96f1dc25f3 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (25m) 26s ago 25m 53.6M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (24m) 26s ago 24m 53.1M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (24m) 26s ago 24m 51.9M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (24m) 26s ago 24m 52.4M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (23m) 6s ago 23m 55.1M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (23m) 6s ago 23m 54.2M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (22m) 6s ago 22m 51.9M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:18:18.725 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (22m) 6s ago 22m 53.5M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:18:18.726 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (16s) 6s ago 21m 53.9M - 2.33.4 514e6a882f6e eb5926cbd0dd 2024-04-26T16:18:18.726 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (20m) 26s ago 20m 75.2M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:18:18.726 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (20m) 6s ago 20m 75.5M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:18:19.156 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:18:19.156 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:18:19.156 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:18:19.156 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:18:19.156 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:18:19.157 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:18:19.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:18:19.172+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:19.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:18:19.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:19.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:19.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:19.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:19.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:19.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:19.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:19.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:18:19.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:18:19.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:18:19.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:18:19.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:19.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:18:19.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:19.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:19.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:19.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:19.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:19.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:18:19.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:19.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:18:19.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:19.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:18:19.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:19.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:19.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:19.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:19.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:19.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:19.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:19.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:19.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:19.536 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:18:19.536 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:18:19.536 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:18:19.536 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-26T16:18:19.536 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [ 2024-04-26T16:18:19.536 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr" 2024-04-26T16:18:19.537 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T16:18:19.537 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "2/2 daemons upgraded", 2024-04-26T16:18:19.537 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-26T16:18:19.537 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:18:19.537 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:18:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='client.25345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/3352074709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:19 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='client.25345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/3352074709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T16:18:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T16:18:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='client.25345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/3352074709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T16:18:19.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:19.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:19.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:19 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: from='client.25357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:18:20.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:18:20.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:20.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: from='client.25357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:20.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:18:20.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:18:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all mgr 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all crash 2024-04-26T16:18:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all mds 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all nfs 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all mgr 2024-04-26T16:18:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all crash 2024-04-26T16:18:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all mds 2024-04-26T16:18:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:18:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:18:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:18:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all nfs 2024-04-26T16:18:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:20 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:18:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: from='client.25357 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:20.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:18:20.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:20.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: Upgrade: Setting container_image for all mgr 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: Upgrade: Setting container_image for all crash 2024-04-26T16:18:20.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: Upgrade: Setting container_image for all mds 2024-04-26T16:18:20.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:18:20.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:18:20.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:18:20.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: Upgrade: Setting container_image for all nfs 2024-04-26T16:18:20.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:20 smithi064 ceph-mon[22742]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:18:21.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:21 smithi064 ceph-mon[22742]: from='client.15453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:21.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:21 smithi059 ceph-mon[26536]: from='client.15453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:21.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:21 smithi059 ceph-mon[21819]: from='client.15453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:21.833 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:21 smithi059 systemd[1]: Stopping Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:18:22.216 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:21 smithi059 podman[72251]: 2024-04-26 16:18:21.926842309 +0000 UTC m=+0.146581755 container died 9b14cfbf9c80879eebd8b8ccfc3d1a7f6e2ba759d7c358e4c429c37e75fc6b6f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:22.549 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:22 smithi059 podman[72251]: 2024-04-26 16:18:22.307529545 +0000 UTC m=+0.527268996 container cleanup 9b14cfbf9c80879eebd8b8ccfc3d1a7f6e2ba759d7c358e4c429c37e75fc6b6f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:22.549 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:22 smithi059 bash[72251]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T16:18:22.550 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:22 smithi059 ceph-mon[26536]: Upgrade: Updating node-exporter.a (1/2) 2024-04-26T16:18:22.550 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:22 smithi059 ceph-mon[26536]: Deploying daemon node-exporter.a on smithi059 2024-04-26T16:18:22.550 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:22 smithi059 ceph-mon[26536]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:22.550 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:22 smithi059 ceph-mon[21819]: Upgrade: Updating node-exporter.a (1/2) 2024-04-26T16:18:22.550 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:22 smithi059 ceph-mon[21819]: Deploying daemon node-exporter.a on smithi059 2024-04-26T16:18:22.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:22 smithi059 ceph-mon[21819]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:22.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:22 smithi064 ceph-mon[22742]: Upgrade: Updating node-exporter.a (1/2) 2024-04-26T16:18:22.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:22 smithi064 ceph-mon[22742]: Deploying daemon node-exporter.a on smithi059 2024-04-26T16:18:22.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:22 smithi064 ceph-mon[22742]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:22.830 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:22 smithi059 podman[72266]: 2024-04-26 16:18:22.550315989 +0000 UTC m=+0.624886298 container remove 9b14cfbf9c80879eebd8b8ccfc3d1a7f6e2ba759d7c358e4c429c37e75fc6b6f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:22.830 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:22 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T16:18:22.830 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:22 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T16:18:22.830 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:22 smithi059 systemd[1]: Stopped Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:18:22.830 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:22 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 5.572s CPU time. 2024-04-26T16:18:23.082 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:22 smithi059 systemd[1]: Starting Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:18:23.501 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:23 smithi059 bash[72394]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-26T16:18:23.832 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:23 smithi059 bash[72394]: Getting image source signatures 2024-04-26T16:18:23.832 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:23 smithi059 bash[72394]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-26T16:18:23.833 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:23 smithi059 bash[72394]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-26T16:18:23.833 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:23 smithi059 bash[72394]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-26T16:18:24.425 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:18:24] "GET /metrics HTTP/1.1" 200 38055 "" "Prometheus/2.33.4" 2024-04-26T16:18:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:24 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:24 smithi064 ceph-mon[22742]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:24.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:24 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:24.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:24 smithi059 ceph-mon[21819]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:24 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:24 smithi059 ceph-mon[26536]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:25.318 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:24 smithi059 bash[72394]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-26T16:18:25.318 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 bash[72394]: Writing manifest to image destination 2024-04-26T16:18:25.582 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 podman[72394]: 2024-04-26 16:18:25.272004477 +0000 UTC m=+2.277550742 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T16:18:25.583 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 podman[72394]: 2024-04-26 16:18:25.36191215 +0000 UTC m=+2.367458403 container create feeb6104cfe1f3e22eaac074b81b8610bc746c37ea9547da271e0666f5ca684d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:25.976 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 podman[72394]: 2024-04-26 16:18:25.614097264 +0000 UTC m=+2.619643517 container init feeb6104cfe1f3e22eaac074b81b8610bc746c37ea9547da271e0666f5ca684d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:25.995 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 podman[72394]: 2024-04-26 16:18:25.617815843 +0000 UTC m=+2.623362342 container start feeb6104cfe1f3e22eaac074b81b8610bc746c37ea9547da271e0666f5ca684d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.622Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.622Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.622Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.623Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.623Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.623Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T16:18:25.996 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T16:18:25.997 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T16:18:25.997 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T16:18:25.998 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T16:18:25.998 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T16:18:25.998 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T16:18:25.998 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T16:18:25.998 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T16:18:25.998 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T16:18:25.998 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T16:18:25.998 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T16:18:25.998 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T16:18:25.999 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T16:18:26.000 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T16:18:26.000 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T16:18:26.000 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T16:18:26.000 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T16:18:26.000 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T16:18:26.000 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T16:18:26.000 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T16:18:26.000 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T16:18:26.000 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a[72445]: ts=2024-04-26T16:18:25.624Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 bash[72394]: feeb6104cfe1f3e22eaac074b81b8610bc746c37ea9547da271e0666f5ca684d 2024-04-26T16:18:26.001 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:18:25 smithi059 systemd[1]: Started Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:18:26.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:26 smithi064 ceph-mon[22742]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:26.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:26 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:26.678 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:26 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:26.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:26 smithi059 ceph-mon[26536]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:26 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:26 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:26 smithi059 ceph-mon[21819]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:18:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:26 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:26 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:28.431 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:28 smithi064 systemd[1]: Stopping Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:18:28.687 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:28 smithi064 podman[59576]: 2024-04-26 16:18:28.501073784 +0000 UTC m=+0.124245611 container died fb96f1dc25f3d2ece31db77d87801cb5a6c2fa9d1341131528278f29ce2df431 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:18:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:28 smithi064 ceph-mon[22742]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:18:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:28 smithi064 ceph-mon[22742]: Upgrade: Updating node-exporter.b (2/2) 2024-04-26T16:18:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:28 smithi064 ceph-mon[22742]: Deploying daemon node-exporter.b on smithi064 2024-04-26T16:18:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:28 smithi059 ceph-mon[21819]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:18:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:28 smithi059 ceph-mon[21819]: Upgrade: Updating node-exporter.b (2/2) 2024-04-26T16:18:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:28 smithi059 ceph-mon[21819]: Deploying daemon node-exporter.b on smithi064 2024-04-26T16:18:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:28 smithi059 ceph-mon[26536]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:18:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:28 smithi059 ceph-mon[26536]: Upgrade: Updating node-exporter.b (2/2) 2024-04-26T16:18:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:28 smithi059 ceph-mon[26536]: Deploying daemon node-exporter.b on smithi064 2024-04-26T16:18:29.149 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:28 smithi064 podman[59576]: 2024-04-26 16:18:28.906739711 +0000 UTC m=+0.529911538 container cleanup fb96f1dc25f3d2ece31db77d87801cb5a6c2fa9d1341131528278f29ce2df431 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:18:29.149 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:28 smithi064 bash[59576]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T16:18:29.437 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:29 smithi064 podman[59591]: 2024-04-26 16:18:29.15006143 +0000 UTC m=+0.649017790 container remove fb96f1dc25f3d2ece31db77d87801cb5a6c2fa9d1341131528278f29ce2df431 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:18:29.437 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:29 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T16:18:29.437 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:29 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T16:18:29.437 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:29 smithi064 systemd[1]: Stopped Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:18:29.437 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:29 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 5.514s CPU time. 2024-04-26T16:18:29.937 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:29 smithi064 systemd[1]: Starting Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:18:29.937 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:29 smithi064 bash[59724]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-26T16:18:30.436 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:30 smithi064 bash[59724]: Getting image source signatures 2024-04-26T16:18:30.436 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:30 smithi064 bash[59724]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-26T16:18:30.436 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:30 smithi064 bash[59724]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-26T16:18:30.436 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:30 smithi064 bash[59724]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-26T16:18:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:30 smithi064 ceph-mon[22742]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:30 smithi059 ceph-mon[21819]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:30 smithi059 ceph-mon[26536]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:31.936 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:31 smithi064 bash[59724]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-26T16:18:31.937 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:31 smithi064 bash[59724]: Writing manifest to image destination 2024-04-26T16:18:32.261 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:31 smithi064 podman[59724]: 2024-04-26 16:18:31.961456224 +0000 UTC m=+2.409495528 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T16:18:32.261 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 podman[59724]: 2024-04-26 16:18:32.059888114 +0000 UTC m=+2.507927401 container create 2d4920f76a2e228c984cb4c9ebe311fefbe0db3e205bcac88d852647b6fa2eff (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:18:32.261 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 podman[59724]: 2024-04-26 16:18:32.261999979 +0000 UTC m=+2.710039267 container init 2d4920f76a2e228c984cb4c9ebe311fefbe0db3e205bcac88d852647b6fa2eff (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:18:32.533 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-mon[22742]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:32.533 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:32.533 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:32.533 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:32.534 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 podman[59724]: 2024-04-26 16:18:32.265939195 +0000 UTC m=+2.713978482 container start 2d4920f76a2e228c984cb4c9ebe311fefbe0db3e205bcac88d852647b6fa2eff (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:18:32.534 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.270Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T16:18:32.534 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.270Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T16:18:32.534 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.271Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T16:18:32.534 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.271Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T16:18:32.534 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.271Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T16:18:32.534 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.271Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T16:18:32.534 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T16:18:32.534 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T16:18:32.535 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T16:18:32.535 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T16:18:32.535 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T16:18:32.535 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T16:18:32.535 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T16:18:32.535 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T16:18:32.535 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T16:18:32.535 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T16:18:32.536 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T16:18:32.536 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T16:18:32.536 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T16:18:32.536 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T16:18:32.536 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T16:18:32.536 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T16:18:32.536 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T16:18:32.536 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T16:18:32.537 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T16:18:32.538 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b[59774]: ts=2024-04-26T16:18:32.272Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T16:18:32.539 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 bash[59724]: 2d4920f76a2e228c984cb4c9ebe311fefbe0db3e205bcac88d852647b6fa2eff 2024-04-26T16:18:32.539 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:18:32 smithi064 systemd[1]: Started Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:18:32.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:32 smithi059 ceph-mon[26536]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:32.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:32 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:32.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:32 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:32.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:32 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:32.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:32 smithi059 ceph-mon[21819]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:32.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:32 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:32.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:32 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:32.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:32 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:33.510 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:33 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:33 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:33.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:33 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:34.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:18:34] "GET /metrics HTTP/1.1" 200 38055 "" "Prometheus/2.33.4" 2024-04-26T16:18:34.774 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:34 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:34.774 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:34 smithi059 ceph-mon[21819]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:34.774 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:34 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:34.775 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:34 smithi059 ceph-mon[26536]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:34.795 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:34 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:34.795 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:34 smithi064 ceph-mon[22742]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:36.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:36 smithi064 ceph-mon[22742]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:36.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:36 smithi059 ceph-mon[21819]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:36.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:36 smithi059 ceph-mon[26536]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:37.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:37 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:37 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:37 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:37 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:37 smithi064 ceph-mon[22742]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:38.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:37 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:37 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:37 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:37 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:37 smithi059 ceph-mon[26536]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:38.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:37 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:37 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:37 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:37 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:37 smithi059 ceph-mon[21819]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:38.910 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:38 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.911 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:38 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.911 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:38 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:38.911 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:38 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:38 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:38 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:38 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:38 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:39.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:38 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:39.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:38 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:39.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:38 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:39.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:38 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:39.848 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:39 smithi064 ceph-mon[22742]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:40.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:39 smithi059 ceph-mon[21819]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:39 smithi059 ceph-mon[26536]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:42.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:42.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:42.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:42.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:42.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:42 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:18:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:18:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-04-26T16:18:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:18:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:18:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:18:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:18:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:42 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:18:43.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:18:43.394+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:43.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:43.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:43.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:43.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:43.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:43.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:43.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:43.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:43.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:18:43.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:43.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:43.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.395 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:18:44] "GET /metrics HTTP/1.1" 200 38055 "" "Prometheus/2.33.4" 2024-04-26T16:18:44.663 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:44.663 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-26T16:18:44.663 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:44.663 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:44.664 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:18:44.665 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:18:44.666 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.667 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:44 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-26T16:18:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:18:44.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:44.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:44.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:44.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:44.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:18:44.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:44.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:44.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:44.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:44.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:18:44.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:18:44.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:18:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:18:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:18:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:18:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:18:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:18:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:18:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:18:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:18:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:18:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:18:44.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:44 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:46.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:46 smithi064 ceph-mon[22742]: Upgrade: Updating prometheus.a 2024-04-26T16:18:46.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:46 smithi064 ceph-mon[22742]: Deploying daemon prometheus.a on smithi064 2024-04-26T16:18:46.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:46 smithi064 ceph-mon[22742]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:46.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:46 smithi059 ceph-mon[26536]: Upgrade: Updating prometheus.a 2024-04-26T16:18:46.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:46 smithi059 ceph-mon[26536]: Deploying daemon prometheus.a on smithi064 2024-04-26T16:18:46.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:46 smithi059 ceph-mon[26536]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:46 smithi059 ceph-mon[21819]: Upgrade: Updating prometheus.a 2024-04-26T16:18:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:46 smithi059 ceph-mon[21819]: Deploying daemon prometheus.a on smithi064 2024-04-26T16:18:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:46 smithi059 ceph-mon[21819]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:48 smithi064 ceph-mon[22742]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-26T16:18:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:48 smithi059 ceph-mon[21819]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-26T16:18:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:48.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:48 smithi059 ceph-mon[26536]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-26T16:18:48.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:18:49.918 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:18:50.610 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (13m) 13s ago 21m 24.2M - 0.23.0 ba2b418f427c ca6d4a23fcc2 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (13m) 13s ago 21m 54.7M - 8.3.5 dad864ee21e9 0d16158d9791 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 13s ago 20m 59.1M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (42s) 13s ago 26m 464M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (10m) 13s ago 27m 513M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (27m) 13s ago 27m 78.4M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (26m) 13s ago 26m 60.8M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (26m) 13s ago 26m 63.0M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (25s) 13s ago 22m 7180k - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (18s) 13s ago 22m 5448k - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (25m) 13s ago 25m 54.5M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (25m) 13s ago 25m 54.2M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:18:50.611 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (24m) 13s ago 24m 52.1M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:18:50.612 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (24m) 13s ago 24m 52.5M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:18:50.612 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (24m) 13s ago 24m 55.0M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:18:50.612 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (23m) 13s ago 23m 54.3M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:18:50.612 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (23m) 13s ago 23m 51.9M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:18:50.612 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (23m) 13s ago 23m 53.4M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:18:50.612 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (48s) 13s ago 22m 61.6M - 2.33.4 514e6a882f6e eb5926cbd0dd 2024-04-26T16:18:50.612 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (20m) 13s ago 20m 75.6M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:18:50.612 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (20m) 13s ago 20m 75.6M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:18:50.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:50 smithi064 ceph-mon[22742]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:50.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:50 smithi059 ceph-mon[21819]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:50.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:50 smithi059 ceph-mon[26536]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:51.032 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:18:51.032 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:18:51.032 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:18:51.032 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:18:51.032 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:18:51.032 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:18:51.032 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:18:51.032 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:18:51.033 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [ 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr" 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "2/2 daemons upgraded", 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading prometheus daemons", 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:18:51.385 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:18:51.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:51 smithi064 ceph-mon[22742]: from='client.25375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:51.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:51 smithi064 ceph-mon[22742]: from='client.25381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:51.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:51 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2770051466' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:51.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:51 smithi059 ceph-mon[21819]: from='client.25375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:51.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:51 smithi059 ceph-mon[21819]: from='client.25381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:51.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:51 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2770051466' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:51.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:51 smithi059 ceph-mon[26536]: from='client.25375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:51.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:51 smithi059 ceph-mon[26536]: from='client.25381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:51.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:51 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2770051466' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:18:52.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:52 smithi064 ceph-mon[22742]: from='client.25387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:52.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:52 smithi064 ceph-mon[22742]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:18:52.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:52 smithi064 ceph-mon[22742]: from='client.15483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:52 smithi059 ceph-mon[21819]: from='client.25387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:52 smithi059 ceph-mon[21819]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:18:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:52 smithi059 ceph-mon[21819]: from='client.15483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:52.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:52 smithi059 ceph-mon[26536]: from='client.25387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:52 smithi059 ceph-mon[26536]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:18:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:52 smithi059 ceph-mon[26536]: from='client.15483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:18:54.412 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:18:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:18:54] "GET /metrics HTTP/1.1" 200 38055 "" "Prometheus/2.33.4" 2024-04-26T16:18:54.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:54 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:54 smithi064 ceph-mon[22742]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:54.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:54 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:54.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:54 smithi059 ceph-mon[21819]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:54 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:18:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:54 smithi059 ceph-mon[26536]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:56.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:56 smithi064 ceph-mon[22742]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:56.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:56 smithi059 ceph-mon[26536]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:56.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:56 smithi059 ceph-mon[21819]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:18:58.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:18:58 smithi064 ceph-mon[22742]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:58.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:18:58 smithi059 ceph-mon[21819]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:18:58.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:18:58 smithi059 ceph-mon[26536]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:00.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:00 smithi064 ceph-mon[22742]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:00.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:00 smithi059 ceph-mon[21819]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:00.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:00 smithi059 ceph-mon[26536]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:02.587 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-mon[22742]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:02.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:02 smithi059 ceph-mon[21819]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:02.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:02 smithi059 ceph-mon[26536]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 systemd[1]: Stopping Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.831Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T16:19:02.842 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-26T16:19:02.843 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[57695]: ts=2024-04-26T16:19:02.832Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-26T16:19:02.843 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:02 smithi064 podman[61275]: 2024-04-26 16:19:02.842129178 +0000 UTC m=+0.106531128 container died eb5926cbd0ddee1b90d203d164316e6fcfc0207d95c0b834c2d9b7dbb523df90 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:03.390 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:03 smithi064 podman[61275]: 2024-04-26 16:19:03.207020179 +0000 UTC m=+0.471422133 container cleanup eb5926cbd0ddee1b90d203d164316e6fcfc0207d95c0b834c2d9b7dbb523df90 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:03.390 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:03 smithi064 bash[61275]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T16:19:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:03.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:03 smithi064 podman[61289]: 2024-04-26 16:19:03.39112582 +0000 UTC m=+0.551132553 container remove eb5926cbd0ddee1b90d203d164316e6fcfc0207d95c0b834c2d9b7dbb523df90 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:03.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:03 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T16:19:03.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:03 smithi064 systemd[1]: Stopped Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:19:03.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:03 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.311s CPU time. 2024-04-26T16:19:03.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:03.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:04.152 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:03 smithi064 systemd[1]: Starting Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:19:04.152 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 podman[61421]: 2024-04-26 16:19:03.945466622 +0000 UTC m=+0.020843955 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T16:19:04.152 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 podman[61421]: 2024-04-26 16:19:04.059364801 +0000 UTC m=+0.134742122 container create 279642636cd094659590087aff8c222754ebe100c627d655eb7ff7c428b97573 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:04.434 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 podman[61421]: 2024-04-26 16:19:04.295178318 +0000 UTC m=+0.370555640 container init 279642636cd094659590087aff8c222754ebe100c627d655eb7ff7c428b97573 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:04.434 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 podman[61421]: 2024-04-26 16:19:04.298125822 +0000 UTC m=+0.373503144 container start 279642636cd094659590087aff8c222754ebe100c627d655eb7ff7c428b97573 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:04.434 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.323Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T16:19:04.434 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.323Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T16:19:04.434 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.323Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi064 (none))" 2024-04-26T16:19:04.434 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.323Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.323Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.324Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.325Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.325Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.325Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.328Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.329Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.331637ms 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.329Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.335Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.349Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.359Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-04-26T16:19:04.435 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.362Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.364Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.366Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.366Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.366Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=47.377µs wal_replay_duration=36.231357ms wbl_replay_duration=114ns total_replay_duration=37.640563ms 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.369Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.369Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.369Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 bash[61421]: 279642636cd094659590087aff8c222754ebe100c627d655eb7ff7c428b97573 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 systemd[1]: Started Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.392Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.318646ms db_storage=829ns remote_storage=1.107µs web_handler=382ns query_engine=695ns scrape=707.828µs scrape_sd=103.293µs notify=21.877µs notify_sd=24.139µs rules=21.065616ms tracing=7.551µs 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.392Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T16:19:04.436 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:19:04.392Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T16:19:04.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:04 smithi064 ceph-mon[22742]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:04.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:04 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:04.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:04 smithi059 ceph-mon[26536]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:04 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:04.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:04 smithi059 ceph-mon[21819]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:05 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:05 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:05 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:05 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:05 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:05 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:05 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:05 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:05 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:06.777 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:06 smithi064 ceph-mon[22742]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:06.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:06 smithi059 ceph-mon[26536]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:06 smithi059 ceph-mon[21819]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:08 smithi064 ceph-mon[22742]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:08.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:08 smithi059 ceph-mon[21819]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:08.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:08 smithi059 ceph-mon[26536]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:10.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:09 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:10.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:09 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:10.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:09 smithi059 ceph-mon[21819]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:10.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:09 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:10.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:09 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:10.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:09 smithi059 ceph-mon[26536]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:10.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:09 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:10.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:09 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:10.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:09 smithi064 ceph-mon[22742]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:11.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:10 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:11.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:10 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:11.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:10 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:11.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:10 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:11.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:10 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:11.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:10 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:12.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:11 smithi064 ceph-mon[22742]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:12.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:11 smithi059 ceph-mon[21819]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:12.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:11 smithi059 ceph-mon[26536]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:14.402 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:19:14] "GET /metrics HTTP/1.1" 200 38053 "" "Prometheus/2.43.0" 2024-04-26T16:19:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:14 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:14 smithi064 ceph-mon[22742]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:14 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:14 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:14 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:14 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:19:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:14 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:14 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:14 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:14 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:14.742 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:14.742 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[26536]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:14.742 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:14.742 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:14.742 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:14.742 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:19:14.742 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[21819]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:14.743 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:14 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:15 smithi064 ceph-mon[22742]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 999 B/s rd, 0 op/s 2024-04-26T16:19:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:15 smithi064 ceph-mon[22742]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-04-26T16:19:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:15 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:19:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:15 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:19:15.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-mon[26536]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 999 B/s rd, 0 op/s 2024-04-26T16:19:15.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-mon[26536]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-04-26T16:19:15.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:19:15.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:19:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-mon[21819]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 999 B/s rd, 0 op/s 2024-04-26T16:19:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-mon[21819]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-04-26T16:19:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:19:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:19:15.651+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:15.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:19:15.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:15.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:15.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:15 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:16.527 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.528 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:16 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:19:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:16.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:16 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:17 smithi064 ceph-mon[22742]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:17 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:19:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:17.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:17 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:17 smithi059 ceph-mon[21819]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:17 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:19:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:19:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:17 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:17 smithi059 ceph-mon[26536]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:17.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:17 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:17.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:19:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:17 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T16:19:18.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:18 smithi064 ceph-mon[22742]: Upgrade: Updating alertmanager.a 2024-04-26T16:19:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:18 smithi064 ceph-mon[22742]: Deploying daemon alertmanager.a on smithi059 2024-04-26T16:19:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:18 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:18.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:18 smithi059 ceph-mon[26536]: Upgrade: Updating alertmanager.a 2024-04-26T16:19:18.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:18 smithi059 ceph-mon[26536]: Deploying daemon alertmanager.a on smithi059 2024-04-26T16:19:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:18 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:18 smithi059 ceph-mon[21819]: Upgrade: Updating alertmanager.a 2024-04-26T16:19:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:18 smithi059 ceph-mon[21819]: Deploying daemon alertmanager.a on smithi059 2024-04-26T16:19:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:18 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:19 smithi064 ceph-mon[22742]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-04-26T16:19:19.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:19 smithi059 ceph-mon[26536]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-04-26T16:19:19.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:19 smithi059 ceph-mon[21819]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-04-26T16:19:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:21 smithi064 ceph-mon[22742]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:21.756 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:19:21.756 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:21 smithi059 ceph-mon[21819]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:21.756 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:21 smithi059 ceph-mon[26536]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:22.454 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:19:22.454 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (14m) 45s ago 22m 24.2M - 0.23.0 ba2b418f427c ca6d4a23fcc2 2024-04-26T16:19:22.454 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (14m) 13s ago 21m 55.3M - 8.3.5 dad864ee21e9 0d16158d9791 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 45s ago 21m 59.1M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (74s) 13s ago 26m 464M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (11m) 45s ago 28m 513M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (28m) 45s ago 28m 78.4M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (26m) 13s ago 26m 61.5M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (27m) 45s ago 27m 63.0M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (57s) 45s ago 22m 7180k - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (50s) 13s ago 22m 15.0M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (26m) 45s ago 26m 54.5M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (25m) 45s ago 25m 54.2M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (25m) 45s ago 25m 52.1M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (25m) 45s ago 25m 52.5M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:19:22.455 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (24m) 13s ago 24m 55.0M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:19:22.456 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (24m) 13s ago 24m 54.3M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:19:22.456 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (23m) 13s ago 23m 52.0M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:19:22.456 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (23m) 13s ago 23m 53.4M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:19:22.456 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (18s) 13s ago 22m 44.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:19:22.456 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (21m) 45s ago 21m 75.6M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:19:22.456 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (21m) 13s ago 21m 75.7M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:19:22.882 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:19:22.882 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:19:22.882 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:19:22.882 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:19:22.882 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:19:22.882 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:19:22.882 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:19:22.882 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:19:22.882 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:19:22.883 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:19:22.883 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:19:22.883 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:19:22.883 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:19:22.883 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:19:22.883 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:19:22.883 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T16:19:22.883 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:19:22.883 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:19:22.883 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:19:23.234 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:19:23.234 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:19:23.234 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:19:23.234 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-26T16:19:23.234 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [ 2024-04-26T16:19:23.234 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr" 2024-04-26T16:19:23.234 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T16:19:23.234 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "2/2 daemons upgraded", 2024-04-26T16:19:23.234 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading alertmanager daemons", 2024-04-26T16:19:23.235 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:19:23.235 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:19:23.547 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:23 smithi059 ceph-mon[26536]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:23.547 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:23 smithi059 ceph-mon[26536]: from='client.25405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:23.547 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:23 smithi059 ceph-mon[26536]: from='client.15495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:23.547 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:23 smithi059 ceph-mon[26536]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:23.547 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:23 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2153883451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:23.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:23 smithi059 ceph-mon[21819]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:23.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:23 smithi059 ceph-mon[21819]: from='client.25405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:23.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:23 smithi059 ceph-mon[21819]: from='client.15495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:23.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:23 smithi059 ceph-mon[21819]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:23.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:23 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2153883451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:23.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:23 smithi064 ceph-mon[22742]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:23 smithi064 ceph-mon[22742]: from='client.25405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:23 smithi064 ceph-mon[22742]: from='client.15495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:23 smithi064 ceph-mon[22742]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:23 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2153883451' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:24.436 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:19:24] "GET /metrics HTTP/1.1" 200 38050 "" "Prometheus/2.43.0" 2024-04-26T16:19:24.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:24 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:24.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:24 smithi059 ceph-mon[21819]: from='client.25429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:24 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:24 smithi059 ceph-mon[26536]: from='client.25429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:24 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:24 smithi064 ceph-mon[22742]: from='client.25429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:25.518 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:25 smithi059 systemd[1]: Stopping Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:19:25.519 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:25 smithi059 ceph-mon[21819]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:19:25.519 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:25 smithi059 ceph-mon[26536]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:19:25.778 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[55997]: level=info ts=2024-04-26T16:19:25.519Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-26T16:19:25.779 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:25 smithi059 podman[74551]: 2024-04-26 16:19:25.570363582 +0000 UTC m=+0.159958904 container died ca6d4a23fcc276a368d9a3e6062baf48ad3a444e8b095c2c8a2f70d45144ec9b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:25 smithi064 ceph-mon[22742]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:19:26.332 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:26 smithi059 podman[74551]: 2024-04-26 16:19:26.045244229 +0000 UTC m=+0.634839602 container cleanup ca6d4a23fcc276a368d9a3e6062baf48ad3a444e8b095c2c8a2f70d45144ec9b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:26.332 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:26 smithi059 bash[74551]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T16:19:26.823 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:26 smithi059 podman[74565]: 2024-04-26 16:19:26.397060002 +0000 UTC m=+0.818581403 container remove ca6d4a23fcc276a368d9a3e6062baf48ad3a444e8b095c2c8a2f70d45144ec9b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:26.823 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:26 smithi059 podman[74565]: 2024-04-26 16:19:26.489378158 +0000 UTC m=+0.910899559 volume remove 567f9b5dc020900898a62ee67c288f54776c0d18df9533d368c564a0b83f8ab2 2024-04-26T16:19:26.823 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:26 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T16:19:26.824 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:26 smithi059 systemd[1]: Stopped Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:19:26.824 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:26 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.550s CPU time. 2024-04-26T16:19:27.082 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:26 smithi059 systemd[1]: Starting Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:19:27.082 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 podman[74714]: 2024-04-26 16:19:27.023717342 +0000 UTC m=+0.099863824 volume create aadaeb58a9677a48a3634a227e0dbff73c4a0af32f3ea5dd5961f24b122fab4a 2024-04-26T16:19:27.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 podman[74714]: 2024-04-26 16:19:26.942370598 +0000 UTC m=+0.018517050 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T16:19:27.384 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 podman[74714]: 2024-04-26 16:19:27.107349412 +0000 UTC m=+0.183495851 container create 70f9197770531f034200b9e1e1679b3a0366c72875d79a7a6ef76e0f8a7edbb7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:27.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:27 smithi059 ceph-mon[26536]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:27 smithi059 ceph-mon[21819]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:27.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 podman[74714]: 2024-04-26 16:19:27.38565966 +0000 UTC m=+0.461806100 container init 70f9197770531f034200b9e1e1679b3a0366c72875d79a7a6ef76e0f8a7edbb7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:27.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 podman[74714]: 2024-04-26 16:19:27.388480944 +0000 UTC m=+0.464627388 container start 70f9197770531f034200b9e1e1679b3a0366c72875d79a7a6ef76e0f8a7edbb7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:19:27.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:19:27.401Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T16:19:27.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:19:27.401Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T16:19:27.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:19:27.402Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.59 port=9094 2024-04-26T16:19:27.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:19:27.404Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T16:19:27.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:19:27.428Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T16:19:27.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:19:27.428Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T16:19:27.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:19:27.432Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T16:19:27.834 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:19:27.432Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T16:19:27.834 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 bash[74714]: 70f9197770531f034200b9e1e1679b3a0366c72875d79a7a6ef76e0f8a7edbb7 2024-04-26T16:19:27.834 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:27 smithi059 systemd[1]: Started Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:19:27.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:27 smithi064 ceph-mon[22742]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:28 smithi064 ceph-mon[22742]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:28 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:28 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:28 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:28.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:28 smithi059 ceph-mon[26536]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:28 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:28 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:28.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:28 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:28 smithi059 ceph-mon[21819]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:28 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:28 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:28.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:28 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:29.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:29 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:19:29.404Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000110662s 2024-04-26T16:19:30.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:30 smithi059 ceph-mon[26536]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:30 smithi059 ceph-mon[21819]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:30.839 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:30 smithi064 ceph-mon[22742]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:33.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:32 smithi064 ceph-mon[22742]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:33.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:32 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:32 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:32 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:32 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:32 smithi059 ceph-mon[21819]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:33.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:32 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:32 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:32 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:32 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:32 smithi059 ceph-mon[26536]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:32 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:32 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:32 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:33.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:32 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:34.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:19:34] "GET /metrics HTTP/1.1" 200 38050 "" "Prometheus/2.43.0" 2024-04-26T16:19:34.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:34 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:34.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:34 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:34.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:34 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:34.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:34 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:34 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:34.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:34 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:34.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:34 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:34.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:34 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:34.340 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:34 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:34.341 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:34 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:34.341 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:34 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:34.341 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:34 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:35.291 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:35 smithi064 ceph-mon[22742]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:35.293 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:35 smithi059 ceph-mon[21819]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:35.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:35 smithi059 ceph-mon[26536]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:37.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[21819]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[26536]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:37.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:19:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:37.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:37 smithi064 ceph-mon[22742]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:37 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:37 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:37 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:37 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:19:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:37 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:37 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:37 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:19:37.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:37 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:19:37.681 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:19:37 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:19:37.407Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003199494s 2024-04-26T16:19:38.657 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:19:38.655+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:38.657 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:19:38.657 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-mon[21819]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 909 B/s rd, 0 op/s 2024-04-26T16:19:38.657 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-mon[21819]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:38.657 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:19:38.657 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:19:38.657 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-mon[26536]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 909 B/s rd, 0 op/s 2024-04-26T16:19:38.658 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-mon[26536]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:38.658 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:19:38.658 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:19:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:38 smithi064 ceph-mon[22742]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 909 B/s rd, 0 op/s 2024-04-26T16:19:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:38 smithi064 ceph-mon[22742]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:38 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:19:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:38 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:19:39.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:39.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:39.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:39.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:39.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:19:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:19:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:19:39.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:19:39.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:39.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:19:38] ENGINE Bus STOPPING 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:19:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:19:38] ENGINE Bus STOPPED 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:19:38] ENGINE Bus STARTING 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:19:38] ENGINE Serving on http://:::9283 2024-04-26T16:19:39.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:19:38] ENGINE Bus STARTED 2024-04-26T16:19:39.564 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:19:39.564 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2024-04-26T16:19:39.564 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.565 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:39 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:39.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:39 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:40 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:19:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:19:40.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:19:40.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:40.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:40 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:19:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:40 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2024-04-26T16:19:40.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:40 smithi064 ceph-mon[22742]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 553 B/s rd, 0 op/s 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:40 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:19:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:19:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:40.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:40.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:40 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:40 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:40 smithi059 ceph-mon[26536]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 553 B/s rd, 0 op/s 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:40 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:40.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:40.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:19:40.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:40 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T16:19:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:40 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2024-04-26T16:19:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:40 smithi059 ceph-mon[21819]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 553 B/s rd, 0 op/s 2024-04-26T16:19:41.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:41 smithi064 ceph-mon[22742]: Upgrade: Updating grafana.a 2024-04-26T16:19:41.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:41 smithi064 ceph-mon[22742]: Deploying daemon grafana.a on smithi064 2024-04-26T16:19:41.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:41 smithi059 ceph-mon[26536]: Upgrade: Updating grafana.a 2024-04-26T16:19:41.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:41 smithi059 ceph-mon[26536]: Deploying daemon grafana.a on smithi064 2024-04-26T16:19:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:41 smithi059 ceph-mon[21819]: Upgrade: Updating grafana.a 2024-04-26T16:19:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:41 smithi059 ceph-mon[21819]: Deploying daemon grafana.a on smithi064 2024-04-26T16:19:42.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:42 smithi064 ceph-mon[22742]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:42.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:42 smithi059 ceph-mon[26536]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:42.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:42 smithi059 ceph-mon[21819]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:44.421 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:19:44] "GET /metrics HTTP/1.1" 200 38050 "" "Prometheus/2.43.0" 2024-04-26T16:19:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:44 smithi064 ceph-mon[22742]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:44 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:44.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:44 smithi059 ceph-mon[26536]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:44 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:44 smithi059 ceph-mon[21819]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:44 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:46.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:46 smithi064 ceph-mon[22742]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:46.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:46 smithi059 ceph-mon[26536]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:46 smithi059 ceph-mon[21819]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:19:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:48 smithi064 ceph-mon[22742]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:19:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:48 smithi059 ceph-mon[21819]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:19:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:48.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:48 smithi059 ceph-mon[26536]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:19:48.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:19:50.205 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:49 smithi064 systemd[1]: Stopping Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:19:50.205 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[46653]: t=2024-04-26T16:19:50+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-26T16:19:50.462 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:50 smithi064 ceph-mon[22742]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:50.463 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:50 smithi064 podman[63506]: 2024-04-26 16:19:50.206259223 +0000 UTC m=+0.109867165 container died 0d16158d97911e189ca131644d6ad8e7a386ce67f02da5ace5048bacc9fc0dd0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.openshift.expose-services=, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, release=236.1648460182, architecture=x86_64, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.buildah.version=1.24.2, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8) 2024-04-26T16:19:50.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:50 smithi059 ceph-mon[21819]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:50.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:50 smithi059 ceph-mon[26536]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:50.846 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:50 smithi064 podman[63506]: 2024-04-26 16:19:50.57056357 +0000 UTC m=+0.474171512 container cleanup 0d16158d97911e189ca131644d6ad8e7a386ce67f02da5ace5048bacc9fc0dd0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, release=236.1648460182, version=8.5, name=ubi8, com.redhat.component=ubi8-container) 2024-04-26T16:19:50.846 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:50 smithi064 bash[63506]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a 2024-04-26T16:19:51.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:50 smithi064 podman[63520]: 2024-04-26 16:19:50.8467753 +0000 UTC m=+0.641983489 container remove 0d16158d97911e189ca131644d6ad8e7a386ce67f02da5ace5048bacc9fc0dd0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, name=ubi8, release=236.1648460182, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, description=Ceph Grafana Container, vendor=Red Hat, Inc., maintainer=Paul Cuzner , io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=) 2024-04-26T16:19:51.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-26T16:19:51.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 systemd[1]: Stopped Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:19:51.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@grafana.a.service: Consumed 5.677s CPU time. 2024-04-26T16:19:51.536 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 systemd[1]: Starting Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:19:51.536 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 podman[63655]: 2024-04-26 16:19:51.317321759 +0000 UTC m=+0.019181465 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-04-26T16:19:51.536 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 podman[63655]: 2024-04-26 16:19:51.423438758 +0000 UTC m=+0.125298528 container create 5d3c77d009e2d42a9b99de0e32eaa72421ab5e89fd773f2fb2d8770c5dd453f3 (image=quay.io/ceph/grafana:9.4.12, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-26T16:19:51.799 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 podman[63655]: 2024-04-26 16:19:51.642561884 +0000 UTC m=+0.344421589 container init 5d3c77d009e2d42a9b99de0e32eaa72421ab5e89fd773f2fb2d8770c5dd453f3 (image=quay.io/ceph/grafana:9.4.12, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-26T16:19:51.800 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 podman[63655]: 2024-04-26 16:19:51.645507358 +0000 UTC m=+0.347367070 container start 5d3c77d009e2d42a9b99de0e32eaa72421ab5e89fd773f2fb2d8770c5dd453f3 (image=quay.io/ceph/grafana:9.4.12, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-26T16:19:51.800 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 bash[63655]: 5d3c77d009e2d42a9b99de0e32eaa72421ab5e89fd773f2fb2d8770c5dd453f3 2024-04-26T16:19:51.800 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 systemd[1]: Started Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:19:52.050 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800546219Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800738994Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800797505Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800807483Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800812172Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800816382Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800820623Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800824595Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800829683Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800834553Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800838451Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-26T16:19:52.051 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800842455Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-26T16:19:52.052 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800847061Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-26T16:19:52.052 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800851086Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-26T16:19:52.052 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800855054Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-26T16:19:52.052 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800858783Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-26T16:19:52.052 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800868671Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-26T16:19:52.052 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.800872789Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-26T16:19:52.053 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=settings t=2024-04-26T16:19:51.80087707Z level=info msg="App mode production" 2024-04-26T16:19:52.053 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=sqlstore t=2024-04-26T16:19:51.800939377Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-26T16:19:52.053 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=sqlstore t=2024-04-26T16:19:51.800959397Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-26T16:19:52.053 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.813251886Z level=info msg="Starting DB migrations" 2024-04-26T16:19:52.053 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.816737988Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.828315947Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.82952048Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.830044989Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.831504623Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.832661904Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.833774681Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.834137365Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.835000973Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.836095923Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.838617985Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.841517551Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.844175338Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.847233306Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-26T16:19:52.054 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.849852768Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.852643515Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.855269988Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.857979345Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.860766543Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.863508522Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.866216133Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.869052569Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.871969429Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.874722397Z level=info msg="Executing migration" id="create alert_image table" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.87753064Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.88034441Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.882971771Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.885669302Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-26T16:19:52.055 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.888353645Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.888453379Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.891193824Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.893751279Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.89426344Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.894619789Z level=info msg="Executing migration" id="create secrets table" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.895093776Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.901461294Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.904148043Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.906760785Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.913261928Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.919749791Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.922466612Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.925155562Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-26T16:19:52.056 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.927910799Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.930737588Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.933542825Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.936074239Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.938830979Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.941506861Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.94417972Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.946921298Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.949675409Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.952332658Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.955010686Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.957693061Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.960112211Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.962885154Z level=info msg="Executing migration" id="add correlation config column" 2024-04-26T16:19:52.057 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.965603055Z level=info msg="Executing migration" id="create entity_events table" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.968284312Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.970951577Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.971048879Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.973504845Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.973586323Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.976305817Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.976776648Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.977614418Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.978510697Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.979359845Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.980125963Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.980914368Z level=info msg="Executing migration" id="Drop public config table" 2024-04-26T16:19:52.058 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.981737973Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.982546829Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.983405563Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.984259831Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.985083192Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.988938624Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.990110569Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.991286448Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.992141928Z level=info msg="Executing migration" id="add share column" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.993345889Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.994127323Z level=info msg="Executing migration" id="create file table" 2024-04-26T16:19:52.059 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.994964661Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.995877476Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.996750123Z level=info msg="Executing migration" id="create file_meta table" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.997613704Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.998479571Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:51 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:51.999294901Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.000528256Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.001368658Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.002226776Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.003456596Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.004333728Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.005171868Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.006008512Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.006866127Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-26T16:19:52.060 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.007718136Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.008928783Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.010135506Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.010986386Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.024279779Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.027047888Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.029931774Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.034463731Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.03714621Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.039971589Z level=info msg="Executing migration" id="create folder table" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.042820894Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.045668004Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.048507727Z level=info msg="Executing migration" id="Update folder title length" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.051315724Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-26T16:19:52.061 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=migrator t=2024-04-26T16:19:52.054153232Z level=info msg="migrations completed" performed=105 skipped=376 duration=237.453394ms 2024-04-26T16:19:52.437 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=sqlstore t=2024-04-26T16:19:52.054685813Z level=info msg="Created default organization" 2024-04-26T16:19:52.437 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=plugin.finder t=2024-04-26T16:19:52.091597934Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-04-26T16:19:52.437 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=secrets t=2024-04-26T16:19:52.091692306Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-26T16:19:52.437 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=query_data t=2024-04-26T16:19:52.095518736Z level=info msg="Query Service initialization" 2024-04-26T16:19:52.437 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=live.push_http t=2024-04-26T16:19:52.097842855Z level=info msg="Live Push Gateway initialization" 2024-04-26T16:19:52.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-mon[22742]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:52.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:52.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:52.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:52.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:52 smithi059 ceph-mon[26536]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:52.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:52 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:52.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:52 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:52.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:52 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:52 smithi059 ceph-mon[21819]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:52 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:52 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:52.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:52 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:19:53.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=infra.usagestats.collector t=2024-04-26T16:19:52.757822981Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-26T16:19:53.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=provisioning.datasources t=2024-04-26T16:19:52.759937801Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-04-26T16:19:53.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=provisioning.datasources t=2024-04-26T16:19:52.759946913Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-04-26T16:19:53.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=provisioning.datasources t=2024-04-26T16:19:52.760052705Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-26T16:19:53.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=provisioning.alerting t=2024-04-26T16:19:52.782778151Z level=info msg="starting to provision alerting" 2024-04-26T16:19:53.188 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=provisioning.alerting t=2024-04-26T16:19:52.782792114Z level=info msg="finished to provision alerting" 2024-04-26T16:19:53.188 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=ngalert.state.manager t=2024-04-26T16:19:52.782923612Z level=info msg="Warming state cache for startup" 2024-04-26T16:19:53.188 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=grafanaStorageLogger t=2024-04-26T16:19:52.783769017Z level=info msg="storage starting" 2024-04-26T16:19:53.188 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=http.server t=2024-04-26T16:19:52.785370807Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-26T16:19:53.188 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=ngalert.state.manager t=2024-04-26T16:19:52.787213235Z level=info msg="State cache has been initialized" states=0 duration=4.288176ms 2024-04-26T16:19:53.188 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=ticker t=2024-04-26T16:19:52.787276338Z level=info msg=starting first_tick=2024-04-26T16:20:00Z 2024-04-26T16:19:53.188 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=ngalert.multiorg.alertmanager t=2024-04-26T16:19:52.787280565Z level=info msg="starting MultiOrg Alertmanager" 2024-04-26T16:19:53.188 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=sqlstore.transactions t=2024-04-26T16:19:52.807329897Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T16:19:53.188 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:19:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=sqlstore.transactions t=2024-04-26T16:19:52.819526558Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-26T16:19:53.618 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:19:54.306 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:19:54.306 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (27s) 22s ago 22m 16.7M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:19:54.306 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 starting - - - - 2024-04-26T16:19:54.306 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 22s ago 21m 61.2M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:19:54.306 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (106s) 22s ago 27m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:19:54.306 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (11m) 22s ago 28m 518M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:19:54.306 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (28m) 22s ago 28m 79.3M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:19:54.306 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (27m) 22s ago 27m 60.7M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:19:54.306 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (27m) 22s ago 27m 59.6M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (89s) 22s ago 23m 8493k - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (82s) 22s ago 23m 17.5M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (26m) 22s ago 26m 54.3M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (26m) 22s ago 26m 54.2M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (26m) 22s ago 26m 52.1M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (25m) 22s ago 25m 52.5M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (25m) 22s ago 25m 55.0M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (24m) 22s ago 24m 54.3M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (24m) 22s ago 24m 52.0M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (24m) 22s ago 24m 53.4M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (50s) 22s ago 23m 55.5M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (21m) 22s ago 21m 75.8M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:19:54.307 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (21m) 22s ago 21m 75.8M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:19:54.438 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:19:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:19:54] "GET /metrics HTTP/1.1" 200 38054 "" "Prometheus/2.43.0" 2024-04-26T16:19:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:54 smithi064 ceph-mon[22742]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:54 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:54.728 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:19:54.728 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:19:54.728 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:19:54.728 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:19:54.728 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:19:54.728 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:19:54.729 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:19:54.730 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:54 smithi059 ceph-mon[26536]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:54.730 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:54 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:54.730 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:54 smithi059 ceph-mon[21819]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:54.730 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:54 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [ 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr" 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "2/2 daemons upgraded", 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading grafana daemons", 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:19:55.079 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:19:55.734 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:55 smithi064 ceph-mon[22742]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:55.734 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:55 smithi064 ceph-mon[22742]: from='client.15525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:55.734 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:55 smithi064 ceph-mon[22742]: from='client.25447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:55.734 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:55 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2232957342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:55 smithi059 ceph-mon[21819]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:55 smithi059 ceph-mon[21819]: from='client.15525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:55 smithi059 ceph-mon[21819]: from='client.25447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:55 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2232957342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:55 smithi059 ceph-mon[26536]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:55 smithi059 ceph-mon[26536]: from='client.15525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:55 smithi059 ceph-mon[26536]: from='client.25447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:55.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:55 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2232957342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:19:56.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:56 smithi059 ceph-mon[21819]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:56.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:56 smithi059 ceph-mon[21819]: from='client.25459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:56.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:56 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:56.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:56 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:56.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:56 smithi059 ceph-mon[26536]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:56.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:56 smithi059 ceph-mon[26536]: from='client.25459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:56.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:56 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:56.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:56 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:56.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:56 smithi064 ceph-mon[22742]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:19:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:56 smithi064 ceph-mon[22742]: from='client.25459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:19:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:56 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:56 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:58.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:58 smithi059 ceph-mon[21819]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:58.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:58 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:58.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:19:58 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:58.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:58 smithi059 ceph-mon[26536]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:58.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:58 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:58.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:19:58 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:58 smithi064 ceph-mon[22742]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:19:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:58 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:19:58.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:19:58 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:00.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:00 smithi064 ceph-mon[22742]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:00.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:00 smithi064 ceph-mon[22742]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:20:00.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:00 smithi064 ceph-mon[22742]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:20:00.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:00 smithi064 ceph-mon[22742]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:20:00.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:00 smithi059 ceph-mon[26536]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:00.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:00 smithi059 ceph-mon[26536]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:20:00.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:00 smithi059 ceph-mon[26536]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:20:00.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:00 smithi059 ceph-mon[26536]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:20:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:00 smithi059 ceph-mon[21819]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:00 smithi059 ceph-mon[21819]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:20:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:00 smithi059 ceph-mon[21819]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:20:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:00 smithi059 ceph-mon[21819]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:20:01.618 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:01.618 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:01.618 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:01.618 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:01.618 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:20:01.618 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:20:01.618 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:01.618 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:01.618 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:01.619 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:01.620 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:01.620 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:01.620 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:01 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:20:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:20:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:01 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:02.638 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:20:02.503+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:02.638 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:20:02.638 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:02.638 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:02.638 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:02.639 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:02.640 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:02.640 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:02.640 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:02.640 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:02.640 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:02.640 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:02.641 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:02.642 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:20:02] ENGINE Bus STOPPING 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:20:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:20:02] ENGINE Bus STOPPED 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:20:02] ENGINE Bus STARTING 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:20:02] ENGINE Serving on http://:::9283 2024-04-26T16:20:02.643 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:20:02] ENGINE Bus STARTED 2024-04-26T16:20:03.658 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:03.658 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:03.658 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:03.658 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:03.659 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:20:03.660 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:03.661 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:03.662 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:03.662 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:03.662 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:03.662 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:03.662 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:03.662 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:03.662 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:03.662 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:03.663 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:03.665 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:03.666 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.670 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.671 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:20:03.671 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:20:03.671 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:20:03.671 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.671 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.671 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:20:03.671 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:20:03.671 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:20:03.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:03.674 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:03.675 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.675 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:03.675 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:03.675 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:03.675 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:03.675 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.675 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:20:03.675 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:20:03.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:20:03.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:20:03.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:20:03.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:20:03.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:03.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:03.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:03.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:03.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:03.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:03.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:03.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:03 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:03.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:03.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:03.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:20:03.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi064.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:20:03.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:03.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:03.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:03.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:03.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:03.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:03 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:04.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:20:04] "GET /metrics HTTP/1.1" 200 38065 "" "Prometheus/2.43.0" 2024-04-26T16:20:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:04 smithi064 ceph-mon[22742]: Upgrade: Finalizing container_image settings 2024-04-26T16:20:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:04 smithi064 ceph-mon[22742]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:20:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:04 smithi064 ceph-mon[22742]: Upgrade: Complete! 2024-04-26T16:20:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:04 smithi064 ceph-mon[22742]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 654 B/s rd, 0 op/s 2024-04-26T16:20:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:04 smithi064 ceph-mon[22742]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 879 B/s rd, 0 op/s 2024-04-26T16:20:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:04 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:04 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:04.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:04 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:04.763 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[26536]: Upgrade: Finalizing container_image settings 2024-04-26T16:20:04.764 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[26536]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:20:04.764 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[26536]: Upgrade: Complete! 2024-04-26T16:20:04.764 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[26536]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 654 B/s rd, 0 op/s 2024-04-26T16:20:04.764 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[26536]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 879 B/s rd, 0 op/s 2024-04-26T16:20:04.764 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:04.764 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:04.764 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:04.764 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[21819]: Upgrade: Finalizing container_image settings 2024-04-26T16:20:04.764 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[21819]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:20:04.765 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[21819]: Upgrade: Complete! 2024-04-26T16:20:04.765 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[21819]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 654 B/s rd, 0 op/s 2024-04-26T16:20:04.765 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[21819]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 879 B/s rd, 0 op/s 2024-04-26T16:20:04.765 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:04.765 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:04.765 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:04.765 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:20:04.484+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:04.765 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:04.766 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:04.767 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:04.768 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:04.769 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.521 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:05.521 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:05.521 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:05.521 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:05.521 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:05.521 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.521 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.521 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:05.521 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:05.521 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:05.522 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:05.523 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:05.524 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:05.525 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:05.526 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.527 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:05.528 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[21819]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[21819]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:05.529 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[26536]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[26536]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:05.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:05 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:05 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:05.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:05.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:05 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:05 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:05 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:05.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:05 smithi064 ceph-mon[22742]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-26T16:20:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:05 smithi064 ceph-mon[22742]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:20:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:05 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:05 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:05 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:05.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:05 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:06.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:06.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:06.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:06.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:06.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:20:06.349+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:06.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:20:06.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:06.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:06.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:06.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:06.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:06.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:06.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:20:06.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:06.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:20:06.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:06.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:06.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:06.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:06.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:06.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:06.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:06.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:06.589 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:06.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:06 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:06.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:06 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:07.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:07 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:07.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:07.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:07.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:07.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:07.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:07.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:07.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:07.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:07.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:20:07.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:07.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:07.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:07.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:07.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:07.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:07.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:07.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:07 smithi064 ceph-mon[22742]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:07 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:07.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:07 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:07.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:20:07.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:07.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:07 smithi059 ceph-mon[26536]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-04-26T16:20:07.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:07 smithi059 ceph-mon[21819]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-04-26T16:20:08.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:08 smithi059 ceph-mon[21819]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-04-26T16:20:08.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:08 smithi059 ceph-mon[26536]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-04-26T16:20:08.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:08 smithi064 ceph-mon[22742]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-04-26T16:20:10.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:10 smithi059 ceph-mon[26536]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:20:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:10 smithi059 ceph-mon[21819]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:20:10.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:10 smithi064 ceph-mon[22742]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:20:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:13 smithi064 ceph-mon[22742]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:20:13.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:13 smithi059 ceph-mon[21819]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:20:13.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:13 smithi059 ceph-mon[26536]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:20:14.420 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:20:14] "GET /metrics HTTP/1.1" 200 38065 "" "Prometheus/2.43.0" 2024-04-26T16:20:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:14 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:14.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:14 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:14 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:15 smithi064 ceph-mon[22742]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:20:15.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:15 smithi059 ceph-mon[26536]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:20:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:15 smithi059 ceph-mon[21819]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:20:16.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:16 smithi059 ceph-mon[21819]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:16 smithi059 ceph-mon[26536]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:16.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:16 smithi064 ceph-mon[22742]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:18 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:18.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:18 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:18.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:18 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:19 smithi064 ceph-mon[22742]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:19.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:19 smithi059 ceph-mon[21819]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:19 smithi059 ceph-mon[26536]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:20.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:20 smithi059 ceph-mon[21819]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:20.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:20 smithi059 ceph-mon[26536]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:20.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:20 smithi064 ceph-mon[22742]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:23 smithi064 ceph-mon[22742]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:23.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:23 smithi059 ceph-mon[21819]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:23 smithi059 ceph-mon[26536]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:24.423 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:20:24] "GET /metrics HTTP/1.1" 200 38064 "" "Prometheus/2.43.0" 2024-04-26T16:20:24.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:24 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:24.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:24 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:24 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:25.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:25 smithi064 ceph-mon[22742]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:25.781 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:25 smithi059 ceph-mon[26536]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:25.781 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:25 smithi059 ceph-mon[21819]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:26.205 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-26T16:20:26.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:26 smithi064 ceph-mon[22742]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:26.738 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:26 smithi059 ceph-mon[21819]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:26.739 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:26 smithi059 ceph-mon[26536]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:27.717 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:27 smithi059 ceph-mon[21819]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:27.718 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:27 smithi059 ceph-mon[26536]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:27 smithi064 ceph-mon[22742]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:28.551 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:20:28.801 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:28 smithi059 ceph-mon[21819]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:28.801 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:28 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/1112981606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:28.801 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:28 smithi059 ceph-mon[26536]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:28.802 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:28 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/1112981606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:28 smithi064 ceph-mon[22742]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:28 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/1112981606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:29.197 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-04-26T16:20:30.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:30 smithi059 ceph-mon[21819]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:30.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:30 smithi059 ceph-mon[26536]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:30 smithi064 ceph-mon[22742]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:31.503 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)" 2024-04-26T16:20:31.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:31 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/147045094' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:31 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/147045094' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:31 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/147045094' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:32.281 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-04-26T16:20:33.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:33 smithi059 ceph-mon[21819]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:33.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:33 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:33.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:33 smithi059 ceph-mon[26536]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:33.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:33 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:33 smithi064 ceph-mon[22742]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:33 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:34.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:20:34] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.43.0" 2024-04-26T16:20:34.462 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:20:34.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:34 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:34.734 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:34 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:34.766 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:34 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:35.034 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-04-26T16:20:35.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:35 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/4088088854' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:35.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:35 smithi064 ceph-mon[22742]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:35.688 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:35 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/4088088854' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:35.688 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:35 smithi059 ceph-mon[26536]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:35.688 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:35 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/4088088854' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:35.688 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:35 smithi059 ceph-mon[21819]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:36.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:36 smithi059 ceph-mon[26536]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:36.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:36 smithi059 ceph-mon[21819]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:36.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:36 smithi064 ceph-mon[22742]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:38 smithi064 ceph-mon[22742]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:38.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:38 smithi059 ceph-mon[26536]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:38.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:38 smithi059 ceph-mon[21819]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:39.270 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:20:39.559 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:39 smithi059 ceph-mon[21819]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:39.560 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:39 smithi059 ceph-mon[26536]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:39.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:39 smithi064 ceph-mon[22742]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:39.865 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade status' 2024-04-26T16:20:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:40 smithi059 ceph-mon[26536]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:40.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:40 smithi059 ceph-mon[21819]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:40.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:40 smithi064 ceph-mon[22742]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:41.850 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:20:41.850 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": null, 2024-04-26T16:20:41.850 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": false, 2024-04-26T16:20:41.850 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "", 2024-04-26T16:20:41.850 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:20:41.850 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": null, 2024-04-26T16:20:41.850 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "", 2024-04-26T16:20:41.851 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:20:41.851 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:20:42.485 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:20:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:43 smithi064 ceph-mon[22742]: from='client.25495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:43 smithi064 ceph-mon[22742]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:43.798 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:43 smithi059 ceph-mon[26536]: from='client.25495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:43.799 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:43 smithi059 ceph-mon[26536]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:43.799 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:43 smithi059 ceph-mon[21819]: from='client.25495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:43.799 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:43 smithi059 ceph-mon[21819]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:44.428 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:20:44] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.43.0" 2024-04-26T16:20:44.638 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:20:44.638 INFO:teuthology.orchestra.run.smithi059.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:20:44.638 INFO:teuthology.orchestra.run.smithi059.stdout: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:20:44.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:44 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:44.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:44 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:44 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:45.231 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-04-26T16:20:45.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:45 smithi059 ceph-mon[26536]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:45.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:45 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2913185212' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:20:45.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:45 smithi059 ceph-mon[21819]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:45.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:45 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2913185212' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:20:45.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:45 smithi064 ceph-mon[22742]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:45.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:45 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/2913185212' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:20:46.819 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:46 smithi059 ceph-mon[21819]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:46.819 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:46 smithi059 ceph-mon[26536]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:46.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:46 smithi064 ceph-mon[22742]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:48 smithi064 ceph-mon[22742]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:48 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:48.746 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:48 smithi059 ceph-mon[21819]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:48.746 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:48.746 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:48 smithi059 ceph-mon[26536]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:48.746 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:20:49.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:49 smithi064 ceph-mon[22742]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "mon", "hosts": "smithi064", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:49.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:49 smithi064 ceph-mon[22742]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:49.804 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:49 smithi059 ceph-mon[26536]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "mon", "hosts": "smithi064", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:49.804 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:49 smithi059 ceph-mon[26536]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:49.804 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:49 smithi059 ceph-mon[21819]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "mon", "hosts": "smithi064", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:49.804 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:49 smithi059 ceph-mon[21819]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:20:49.857 INFO:teuthology.orchestra.run.smithi059.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:20:50.475 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-26T16:20:51.080 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[21819]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:20:51.080 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:51.080 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:51.080 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[21819]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:51.080 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:51.080 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:51.080 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[26536]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[26536]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:51.081 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:51.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:51.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:50 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:50 smithi064 ceph-mon[22742]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:20:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:50 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:50 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:50 smithi064 ceph-mon[22742]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:50 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:50 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:20:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:50 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:50 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:50 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:20:51.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:50 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:51 smithi059 ceph-mon[21819]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1009 B/s rd, 0 op/s 2024-04-26T16:20:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:51 smithi059 ceph-mon[21819]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:51 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:51 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:52.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:51 smithi059 ceph-mon[26536]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1009 B/s rd, 0 op/s 2024-04-26T16:20:52.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:51 smithi059 ceph-mon[26536]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:52.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:51 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:52.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:51 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:51 smithi064 ceph-mon[22742]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1009 B/s rd, 0 op/s 2024-04-26T16:20:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:51 smithi064 ceph-mon[22742]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:51 smithi064 ceph-mon[22742]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:20:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:51 smithi064 ceph-mon[22742]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:20:53.237 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:20:53.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:20:53.658+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:53.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:20:53.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:53.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:53.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:53.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:53.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:53.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:53.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:20:53.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:53.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:20:53.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:53.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:20:53.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:53.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:53.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:53.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:53.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:53.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:53.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:53.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:53.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:53.945 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (86s) 82s ago 23m 16.7M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (62s) 57s ago 23m 49.0M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 82s ago 22m 61.2M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (2m) 57s ago 28m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (12m) 82s ago 29m 518M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (29m) 82s ago 29m 79.3M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (28m) 57s ago 28m 60.7M 2048M 17.2.0 e1d6a67b021e cb2d5179017d 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (28m) 82s ago 28m 59.6M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (2m) 82s ago 24m 8493k - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (2m) 57s ago 24m 16.8M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (27m) 82s ago 27m 54.3M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (27m) 82s ago 27m 54.2M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:20:53.946 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (27m) 82s ago 27m 52.1M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:20:53.947 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (26m) 82s ago 26m 52.5M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:20:53.947 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (26m) 57s ago 26m 55.0M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:20:53.947 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (25m) 57s ago 25m 54.3M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:20:53.947 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (25m) 57s ago 25m 52.0M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:20:53.947 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (25m) 57s ago 25m 53.4M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:20:53.947 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (109s) 57s ago 24m 56.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:20:53.947 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (22m) 82s ago 22m 75.8M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:20:53.947 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (22m) 57s ago 22m 75.9M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:20:54.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:20:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:20:54] "GET /metrics HTTP/1.1" 200 38066 "" "Prometheus/2.43.0" 2024-04-26T16:20:54.411 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:20:54.411 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:20:54.411 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-26T16:20:54.411 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:20:54.411 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:20:54.411 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:20:54.411 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:20:54.411 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:20:54.411 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:20:54.411 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:20:54.412 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:20:54.412 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:20:54.412 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:20:54.412 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:20:54.412 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:20:54.412 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-26T16:20:54.412 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:20:54.412 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:20:54.412 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:20:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:54 smithi064 ceph-mon[22742]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-26T16:20:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:54 smithi064 ceph-mon[22742]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:54 smithi064 ceph-mon[22742]: from='client.25519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:54.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:54 smithi064 ceph-mon[22742]: from='client.? 172.21.15.59:0/964204376' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:54.767 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:20:54.767 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", 2024-04-26T16:20:54.767 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:20:54.767 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi064", 2024-04-26T16:20:54.767 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:20:54.767 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "", 2024-04-26T16:20:54.767 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 image", 2024-04-26T16:20:54.767 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:20:54.768 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:20:54.768 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:54 smithi059 ceph-mon[21819]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-26T16:20:54.768 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:54 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:54.768 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:54 smithi059 ceph-mon[21819]: from='client.25519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:54.768 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:54 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/964204376' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:54.768 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:54 smithi059 ceph-mon[26536]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-26T16:20:54.768 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:54 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:20:54.768 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:54 smithi059 ceph-mon[26536]: from='client.25519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:54.769 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:54 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/964204376' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:55 smithi064 ceph-mon[22742]: from='client.25525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:55 smithi064 ceph-mon[22742]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:55.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:55.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:55 smithi064 ceph-mon[22742]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:20:55.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:55 smithi064 ceph-mon[22742]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:55.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:55 smithi059 ceph-mon[26536]: from='client.25525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:55.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:55 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:55.713 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:55.713 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:55.713 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:55.713 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:55.713 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:55.714 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:55.715 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:55.716 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:55 smithi059 ceph-mon[26536]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:20:55.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:55 smithi059 ceph-mon[26536]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:55 smithi059 ceph-mon[21819]: from='client.25525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:55 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:55.718 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:20:55.719 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:20:55.720 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:20:55.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:20:55.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:20:55.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:20:55.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:20:55.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:20:55.722 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:55 smithi059 ceph-mon[21819]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:20:55.723 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:55 smithi059 ceph-mon[21819]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:56 smithi064 ceph-mon[22742]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:56 smithi064 ceph-mon[22742]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:56 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:56 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:56 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:56 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:56 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T16:20:56.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:56 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T16:20:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[26536]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[26536]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T16:20:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T16:20:56.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[21819]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:56.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[21819]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:20:56.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:56.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:20:56.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:20:56.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:56.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T16:20:56.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:56 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T16:20:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:57 smithi064 ceph-mon[22742]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:20:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:57 smithi064 ceph-mon[22742]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:20:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:57 smithi064 ceph-mon[22742]: Upgrade: Setting container_image for all mgr 2024-04-26T16:20:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:57 smithi064 ceph-mon[22742]: Upgrade: It appears safe to stop mon.b 2024-04-26T16:20:57.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:57 smithi059 ceph-mon[26536]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:20:57.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:57 smithi059 ceph-mon[26536]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:20:57.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:57 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all mgr 2024-04-26T16:20:57.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:57 smithi059 ceph-mon[26536]: Upgrade: It appears safe to stop mon.b 2024-04-26T16:20:57.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:57 smithi059 ceph-mon[21819]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:20:57.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:57 smithi059 ceph-mon[21819]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:20:57.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:57 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all mgr 2024-04-26T16:20:57.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:57 smithi059 ceph-mon[21819]: Upgrade: It appears safe to stop mon.b 2024-04-26T16:20:58.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:58 smithi064 ceph-mon[22742]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:58.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:58 smithi064 ceph-mon[22742]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:58.760 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:58 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:20:58.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:58 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:20:58.761 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:58 smithi064 ceph-mon[22742]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:58 smithi059 ceph-mon[21819]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:58 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:58 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:20:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:58 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:20:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:58 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:58.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:58 smithi059 ceph-mon[26536]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:20:58.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:58 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:20:58.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:58 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:20:58.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:58 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:20:58.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:58 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:20:59.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:59 smithi064 systemd[1]: Stopping Ceph mon.b for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:20:59.806 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:59 smithi064 ceph-mon[22742]: Upgrade: Updating mon.b 2024-04-26T16:20:59.806 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:59 smithi064 ceph-mon[22742]: Deploying daemon mon.b on smithi064 2024-04-26T16:20:59.806 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:59 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b[22738]: 2024-04-26T16:20:59.539+0000 7f4e2f0e8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:20:59.807 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:59 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b[22738]: 2024-04-26T16:20:59.539+0000 7f4e2f0e8700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-26T16:20:59.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:59 smithi059 ceph-mon[21819]: Upgrade: Updating mon.b 2024-04-26T16:20:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:20:59 smithi059 ceph-mon[21819]: Deploying daemon mon.b on smithi064 2024-04-26T16:20:59.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:59 smithi059 ceph-mon[26536]: Upgrade: Updating mon.b 2024-04-26T16:20:59.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:20:59 smithi059 ceph-mon[26536]: Deploying daemon mon.b on smithi064 2024-04-26T16:21:00.114 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:20:59 smithi064 podman[65105]: 2024-04-26 16:20:59.805846896 +0000 UTC m=+0.368708884 container died cb2d5179017db3932cd46a59cb85a502f6206767eb9752f5e95f921b12b52c55 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, io.openshift.expose-services=, GIT_CLEAN=True, version=8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, ceph=True, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-26T16:21:00.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:00 smithi064 podman[65105]: 2024-04-26 16:21:00.11462817 +0000 UTC m=+0.677490166 container cleanup cb2d5179017db3932cd46a59cb85a502f6206767eb9752f5e95f921b12b52c55 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, name=centos-stream, release=754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-26T16:21:00.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:00 smithi064 bash[65105]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b 2024-04-26T16:21:00.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:00 smithi064 podman[65119]: 2024-04-26 16:21:00.349326695 +0000 UTC m=+0.540846190 container remove cb2d5179017db3932cd46a59cb85a502f6206767eb9752f5e95f921b12b52c55 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.19.8, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, name=centos-stream, vendor=Red Hat, Inc., vcs-type=git, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-26T16:21:00.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:00 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-26T16:21:00.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:00 smithi064 systemd[1]: Stopped Ceph mon.b for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:21:00.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:00 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.b.service: Consumed 19.035s CPU time. 2024-04-26T16:21:00.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:00 smithi064 systemd[1]: Starting Ceph mon.b for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:21:01.270 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 podman[65252]: 2024-04-26 16:21:00.918906617 +0000 UTC m=+0.018624501 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:21:01.270 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 podman[65252]: 2024-04-26 16:21:01.042265465 +0000 UTC m=+0.141983336 container create 71107a400e689f0ee4c7a9f9d3845b619f6054a47e399ffefcd66a98611e64a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:21:01.539 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 podman[65252]: 2024-04-26 16:21:01.271010414 +0000 UTC m=+0.370728284 container init 71107a400e689f0ee4c7a9f9d3845b619f6054a47e399ffefcd66a98611e64a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:21:01.539 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 podman[65252]: 2024-04-26 16:21:01.274704264 +0000 UTC m=+0.374422138 container start 71107a400e689f0ee4c7a9f9d3845b619f6054a47e399ffefcd66a98611e64a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:21:01.539 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T16:21:01.539 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev), process ceph-mon, pid 2 2024-04-26T16:21:01.539 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: pidfile_write: ignore empty --pid-file 2024-04-26T16:21:01.539 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: load: jerasure load: lrc 2024-04-26T16:21:01.539 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: RocksDB version: 7.9.2 2024-04-26T16:21:01.539 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Git sha 0 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Compile date 2024-04-26 03:22:31 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: DB SUMMARY 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: DB Session ID: 0XY9GN86YQUBK4FRYMPH 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: CURRENT file: CURRENT 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2836 Bytes 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 3346880 ; 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.error_if_exists: 0 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.create_if_missing: 0 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.paranoid_checks: 1 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T16:21:01.540 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T16:21:01.541 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T16:21:01.541 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.env: 0x55df65305cc0 2024-04-26T16:21:01.541 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T16:21:01.541 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.info_log: 0x55df6645dd80 2024-04-26T16:21:01.541 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T16:21:01.542 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.statistics: (nil) 2024-04-26T16:21:01.542 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.use_fsync: 0 2024-04-26T16:21:01.542 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_log_file_size: 0 2024-04-26T16:21:01.542 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T16:21:01.542 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T16:21:01.542 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.allow_fallocate: 1 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.use_direct_reads: 0 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.db_log_dir: 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.wal_dir: 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T16:21:01.543 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.write_buffer_manager: 0x55df6642f0e0 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.unordered_write: 0 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T16:21:01.544 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.row_cache: None 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.wal_filter: None 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.two_write_queues: 0 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.wal_compression: 0 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.atomic_flush: 0 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.log_readahead_size: 0 2024-04-26T16:21:01.545 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_background_jobs: 2 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_background_compactions: -1 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_subcompactions: 1 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T16:21:01.546 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_open_files: -1 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_background_flushes: -1 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Compression algorithms supported: 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: kZSTD supported: 0 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: kXpressCompression supported: 0 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T16:21:01.547 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: kZlibCompression supported: 1 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: kSnappyCompression supported: 1 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: kLZ4Compression supported: 1 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: kBZip2Compression supported: 0 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.merge_operator: 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_filter: None 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T16:21:01.548 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55df6645df40) 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: cache_index_and_filter_blocks: 1 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: pin_top_level_index_and_filter: 1 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_type: 0 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: data_block_index_type: 0 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_shortening: 1 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: checksum: 4 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: no_block_cache: 0 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache: 0x55df664611f0 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_name: BinnedLRUCache 2024-04-26T16:21:01.549 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_options: 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: capacity : 536870912 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: num_shard_bits : 4 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: strict_capacity_limit : 0 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: high_pri_pool_ratio: 0.000 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_cache_compressed: (nil) 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: persistent_cache: (nil) 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_size: 4096 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_size_deviation: 10 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_restart_interval: 16 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: index_block_restart_interval: 1 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: metadata_block_size: 4096 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: partition_filters: 0 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_delta_encoding: 1 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: filter_policy: bloomfilter 2024-04-26T16:21:01.550 INFO:journalctl@ceph.mon.b.smithi064.stdout: whole_key_filtering: 1 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout: verify_compression: 0 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout: read_amp_bytes_per_bit: 0 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout: format_version: 5 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout: enable_index_compression: 1 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout: block_align: 0 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout: max_auto_readahead_size: 262144 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout: prepopulate_block_cache: 0 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout: initial_auto_readahead_size: 8192 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compression: NoCompression 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T16:21:01.551 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.num_levels: 7 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T16:21:01.552 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T16:21:01.553 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T16:21:01.554 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T16:21:01.555 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.inplace_update_support: 0 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.bloom_locality: 0 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.max_successive_merges: 0 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T16:21:01.556 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.ttl: 2592000 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.enable_blob_files: false 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.min_blob_size: 0 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T16:21:01.557 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28878, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: effdba0f-4e95-446a-aa18-349fcc718a59 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714148461304707, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714148461313586, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 2078828, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28875, "largest_seqno": 30815, "table_properties": {"data_size": 2071267, "index_size": 4599, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1925, "raw_key_size": 18340, "raw_average_key_size": 24, "raw_value_size": 2055372, "raw_average_value_size": 2722, "num_data_blocks": 206, "num_entries": 755, "num_filter_entries": 755, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714148461, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "effdba0f-4e95-446a-aa18-349fcc718a59", "db_session_id": "0XY9GN86YQUBK4FRYMPH", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714148461313687, "job": 1, "event": "recovery_finished"} 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T16:21:01.558 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55df6657e000 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: DB pointer 0x55df6656a000 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** DB Stats ** 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-04-26T16:21:01.559 INFO:journalctl@ceph.mon.b.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: L0 1/0 1.98 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 551.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: L6 1/0 9.03 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: Sum 2/0 11.01 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 551.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 551.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** Compaction Stats [default] ** 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 551.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T16:21:01.560 INFO:journalctl@ceph.mon.b.smithi064.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: Cumulative compaction: 0.00 GB write, 150.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: Interval compaction: 0.00 GB write, 150.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: Block cache BinnedLRUCache@0x55df664611f0#2 capacity: 512.00 MB usage: 6.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: starting mon.b rank 2 at public addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] at bind addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: mon.b@-1(???) e3 preinit fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:21:01.561 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: mon.b@-1(???).mds e1 new map 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: mon.b@-1(???).mds e1 print_map 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout: e1 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout: legacy client fscid: -1 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout: 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout: No filesystems configured 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: mon.b@-1(???).osd e106 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: mon.b@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: mon.b@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: mon.b@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: mon.b@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 bash[65252]: 71107a400e689f0ee4c7a9f9d3845b619f6054a47e399ffefcd66a98611e64a1 2024-04-26T16:21:01.562 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 systemd[1]: Started Ceph mon.b for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:21:01.563 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:01 smithi064 ceph-mon[65271]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-26T16:21:01.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:01 smithi059 ceph-mon[21819]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-26T16:21:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:01 smithi059 ceph-mon[26536]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-26T16:21:02.666 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:21:02.666 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: mon.b calling monitor election 2024-04-26T16:21:02.667 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: mon.c calling monitor election 2024-04-26T16:21:02.667 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: mon.a calling monitor election 2024-04-26T16:21:02.667 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T16:21:02.667 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T16:21:02.667 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: fsmap 2024-04-26T16:21:02.667 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T16:21:02.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: mgrmap e42: y(active, since 3m), standbys: x 2024-04-26T16:21:02.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:21:02.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:21:02.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:21:02.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:02.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:02.668 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:02 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:21:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: mon.b calling monitor election 2024-04-26T16:21:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: mon.c calling monitor election 2024-04-26T16:21:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: mon.a calling monitor election 2024-04-26T16:21:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T16:21:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T16:21:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: fsmap 2024-04-26T16:21:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T16:21:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: mgrmap e42: y(active, since 3m), standbys: x 2024-04-26T16:21:02.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:21:02.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:21:02.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:21:02.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:02.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:02.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:02.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:21:02.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: mon.b calling monitor election 2024-04-26T16:21:02.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: mon.c calling monitor election 2024-04-26T16:21:02.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: mon.a calling monitor election 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: fsmap 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: mgrmap e42: y(active, since 3m), standbys: x 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:02.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:02 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:03 smithi059 ceph-mon[21819]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:03 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:03 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:03.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:03 smithi059 ceph-mon[26536]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:03.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:03 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:03.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:03 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:03.845 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:03 smithi064 ceph-mon[65271]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:03.845 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:03 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:03.845 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:03 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:04.535 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:21:04] "GET /metrics HTTP/1.1" 200 38005 "" "Prometheus/2.43.0" 2024-04-26T16:21:06.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:05 smithi059 ceph-mon[26536]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:06.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:05 smithi059 ceph-mon[21819]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:06.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:05 smithi064 ceph-mon[65271]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:06.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:06.323+0000 7f47109cb700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-26T16:21:07.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:06 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:07.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:06 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:07.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:06 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:07.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:06 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:07.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:06 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:07.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:06 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:08.284 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:07 smithi064 ceph-mon[65271]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:08.284 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:07 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:08.284 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:07 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:08.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:07 smithi059 ceph-mon[21819]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:08.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:07 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:08.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:07 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:08.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:07 smithi059 ceph-mon[26536]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:08.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:07 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:08.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:07 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:10.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:09 smithi059 ceph-mon[21819]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:10.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:10 smithi059 ceph-mon[26536]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:10.333 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:09 smithi064 ceph-mon[65271]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:11.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: Detected new or changed devices on smithi064 2024-04-26T16:21:11.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:11.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:11.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:11.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:11.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:21:11.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:21:11.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:11.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:11.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: Detected new or changed devices on smithi064 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:11.485 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:11 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: Detected new or changed devices on smithi064 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:11 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:12.396 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:12.306+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:12.396 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:21:12.396 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:12.396 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:12.397 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:12.398 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:12.398 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:12.398 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:12.398 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:12.398 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:12.398 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:12.398 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:12.399 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:12.400 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:12.401 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:12.401 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:12.401 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:21:12.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:12 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:21:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:12.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T16:21:12.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T16:21:12.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:21:12.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:12 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:21:13.567 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:13.568 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:13.568 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:13.568 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:13.568 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:13.568 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.568 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.569 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:13.569 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:13.569 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:13.569 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:13.569 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:13.569 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:13.570 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:13.570 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:13.570 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:13.570 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:13.570 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:13.570 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:13.571 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.571 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.571 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:13.571 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:13.571 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:13.571 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:13.571 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:13.572 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:13.572 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:13.572 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:13.572 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:21:13.572 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:13.572 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:21:13.573 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:13.573 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:21:13.573 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:13.573 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:13.573 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:13.573 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:13.574 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:13.574 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:13.574 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:13.574 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:13.574 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:13.574 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.575 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.575 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:13.575 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:13.575 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:13.575 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:13.575 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:13.575 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:13.575 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:13.575 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:13.576 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:13.576 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:13.577 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:13.577 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:13.577 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:13.577 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:13.577 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.577 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.578 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:13.578 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:13.578 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:13.578 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:13.578 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:13.578 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:13.579 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:13.579 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:13.579 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:13.579 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:13.579 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:13.579 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:13.580 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.580 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.580 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:13.580 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:13.580 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:13.580 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:13.581 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:13.581 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:13.581 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:13.581 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:13.581 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:21:13.581 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:13.582 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:21:13.582 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:13.582 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:21:13.582 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:13.582 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:13.582 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:13.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:13.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:13.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:13.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:13.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:13.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:13.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:13.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:13.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:13.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:13.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:13.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:13.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all crash 2024-04-26T16:21:13.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all mds 2024-04-26T16:21:13.586 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:21:13.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:21:13.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:21:13.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all nfs 2024-04-26T16:21:13.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:21:13.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all node-exporter 2024-04-26T16:21:13.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all prometheus 2024-04-26T16:21:13.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all alertmanager 2024-04-26T16:21:13.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all grafana 2024-04-26T16:21:13.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all loki 2024-04-26T16:21:13.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all promtail 2024-04-26T16:21:13.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Finalizing container_image settings 2024-04-26T16:21:13.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:21:13.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:21:13.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:21:13.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:21:13.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:21:13.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:21:13.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:21:13.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:21:13.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:13.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all crash 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all mds 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all nfs 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all node-exporter 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all prometheus 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all alertmanager 2024-04-26T16:21:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all grafana 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all loki 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all promtail 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Finalizing container_image settings 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:21:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:21:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:21:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:21:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:21:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:21:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:21:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:21:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:21:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:21:13.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:21:13.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:21:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:21:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:21:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:21:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:21:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:21:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:21:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:21:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: Upgrade: Complete! 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 654 B/s rd, 0 op/s 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:13.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:13.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:13.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:13.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: Upgrade: Complete! 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 654 B/s rd, 0 op/s 2024-04-26T16:21:13.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-04-26T16:21:13.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:13.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:13.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:13.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:13.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:13 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:13.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:21:13.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:13.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all crash 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mds 2024-04-26T16:21:13.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all nfs 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all node-exporter 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all prometheus 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all alertmanager 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all grafana 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all loki 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all promtail 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Finalizing container_image settings 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:21:13.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:21:13.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:21:13.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:21:13.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:21:13.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: Upgrade: Complete! 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:13.696 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:13.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:13.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 654 B/s rd, 0 op/s 2024-04-26T16:21:13.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-04-26T16:21:13.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:13.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:13.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:13.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:13.697 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:13 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:14.259 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:21:14] "GET /metrics HTTP/1.1" 200 38005 "" "Prometheus/2.43.0" 2024-04-26T16:21:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:14.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:14.336+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:14.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:14.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:14.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:14.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:14.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:14.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:14.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:14.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:21:14.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:14.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:21:14.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:14.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:14.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:14.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:14.589 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:14.589 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:14.589 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:14.589 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:14.589 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:14.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:14 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:14 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:14 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:14 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:14 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:14 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:14 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:15 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:21:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:15 smithi064 ceph-mon[65271]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:15 smithi064 ceph-mon[65271]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:15 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:15 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:15 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:15 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:15.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:15 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:15.703 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:15.704 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:15.705 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:15.706 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:15.707 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:15.708 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:15.708 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:15.708 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:15.708 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:15.708 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:15.708 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.708 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.708 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:15.708 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:15.708 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:15.709 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:15.710 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:15.711 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[21819]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[21819]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:15.712 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[26536]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[26536]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:15.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:15.714 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:15 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:16.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:16.100+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:16.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:21:16.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:16.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:16.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:16.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:16.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:16.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:16.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:16.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:16.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:16.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:16.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:16.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:16.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:16.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:16.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:16.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:16.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:16.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:16.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:16.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:17 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:21:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:17 smithi064 ceph-mon[65271]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-04-26T16:21:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:17 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:17.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:21:17.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:21:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:17.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:21:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:17.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:17.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:17.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:17.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:17.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:17.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:17.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:17.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:17.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:17.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:17.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:17.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:17.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:17.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:17 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:17.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:17.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:17.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:17.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:17.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:17.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:17.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:17.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:17 smithi059 ceph-mon[21819]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-04-26T16:21:17.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:17.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:17.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:17.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:17.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:17.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:17 smithi059 ceph-mon[26536]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-04-26T16:21:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:18 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:18.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:18 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:18 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:19 smithi064 ceph-mon[65271]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-04-26T16:21:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:19 smithi059 ceph-mon[26536]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-04-26T16:21:19.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:19 smithi059 ceph-mon[21819]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-04-26T16:21:21.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:21 smithi059 ceph-mon[21819]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:21:21.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:21 smithi059 ceph-mon[26536]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:21:21.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:21 smithi064 ceph-mon[65271]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:21:23.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:23 smithi059 ceph-mon[21819]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:21:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:23 smithi059 ceph-mon[26536]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:21:23.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:23 smithi064 ceph-mon[65271]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:21:24.447 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:21:24] "GET /metrics HTTP/1.1" 200 38074 "" "Prometheus/2.43.0" 2024-04-26T16:21:24.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:24 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:24.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:24 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:24.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:24 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:25.766 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:25 smithi059 ceph-mon[21819]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:21:25.767 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:25 smithi059 ceph-mon[26536]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:21:25.798 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:21:25.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:25 smithi064 ceph-mon[65271]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:21:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:26 smithi059 ceph-mon[26536]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:26 smithi059 ceph-mon[21819]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:26 smithi064 ceph-mon[65271]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:27 smithi059 ceph-mon[21819]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:27.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:27 smithi059 ceph-mon[26536]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (2m) 116s ago 24m 16.7M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (96s) 22s ago 23m 77.5M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 116s ago 23m 61.2M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (3m) 22s ago 28m 466M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (13m) 116s ago 30m 518M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (30m) 116s ago 30m 79.3M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (26s) 22s ago 28m 22.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (29m) 116s ago 29m 59.6M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (3m) 116s ago 24m 8493k - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:21:27.910 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (2m) 22s ago 24m 16.9M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (28m) 116s ago 28m 54.3M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (28m) 116s ago 28m 54.2M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (27m) 116s ago 27m 52.1M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (27m) 116s ago 27m 52.5M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (26m) 22s ago 26m 55.3M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (26m) 22s ago 26m 54.4M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (26m) 22s ago 26m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (25m) 22s ago 25m 53.4M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (2m) 22s ago 25m 62.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (23m) 116s ago 23m 75.8M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:21:27.911 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (23m) 22s ago 23m 76.3M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:21:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:27 smithi064 ceph-mon[65271]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:28.593 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-04-26T16:21:29.802 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:29 smithi059 ceph-mon[21819]: from='client.25555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:29.803 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:29 smithi059 ceph-mon[21819]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:29.803 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:29 smithi059 ceph-mon[26536]: from='client.25555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:29.803 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:29 smithi059 ceph-mon[26536]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:29.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:29 smithi064 ceph-mon[65271]: from='client.25555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:29.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:29 smithi064 ceph-mon[65271]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:30.767 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:21:31.391 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade status' 2024-04-26T16:21:31.707 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:31 smithi059 ceph-mon[26536]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:31.708 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:31 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/2199002251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:31.708 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:31 smithi059 ceph-mon[21819]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:31.708 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:31 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/2199002251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:31.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:31 smithi064 ceph-mon[65271]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:31 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2199002251' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:33.427 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:21:33.427 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": null, 2024-04-26T16:21:33.427 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": false, 2024-04-26T16:21:33.427 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "", 2024-04-26T16:21:33.427 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:21:33.427 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": null, 2024-04-26T16:21:33.428 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "", 2024-04-26T16:21:33.428 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:21:33.428 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:21:33.736 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:33 smithi059 ceph-mon[21819]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:33.736 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:33 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:33.736 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:33 smithi059 ceph-mon[26536]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:33.736 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:33 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:33 smithi064 ceph-mon[65271]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:33.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:33 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:34.067 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:21:34.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:21:34] "GET /metrics HTTP/1.1" 200 38073 "" "Prometheus/2.43.0" 2024-04-26T16:21:34.690 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:34 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:34.690 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:34 smithi059 ceph-mon[26536]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:34.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:34 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:34.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:34 smithi059 ceph-mon[21819]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:34.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:34 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:34.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:34 smithi064 ceph-mon[65271]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:35 smithi059 ceph-mon[21819]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:35 smithi059 ceph-mon[26536]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:35.857 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:35 smithi064 ceph-mon[65271]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:36.344 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:21:36.344 INFO:teuthology.orchestra.run.smithi059.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:21:36.345 INFO:teuthology.orchestra.run.smithi059.stdout: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:21:36.651 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:36 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/771582920' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:21:36.651 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:36 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/771582920' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:21:36.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:36 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/771582920' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:21:36.935 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-04-26T16:21:37.539 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:37 smithi059 ceph-mon[26536]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:37.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:37 smithi059 ceph-mon[21819]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:37.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:37 smithi064 ceph-mon[65271]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:39.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:39 smithi059 ceph-mon[26536]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:39 smithi059 ceph-mon[21819]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:39 smithi064 ceph-mon[65271]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:21:40.794 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:40 smithi059 ceph-mon[21819]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:40.794 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:40 smithi059 ceph-mon[21819]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "mon", "hosts": "smithi059", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:40.794 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:40 smithi059 ceph-mon[26536]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:40.794 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:40 smithi059 ceph-mon[26536]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "mon", "hosts": "smithi059", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:40.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:40 smithi064 ceph-mon[65271]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:40 smithi064 ceph-mon[65271]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "mon", "hosts": "smithi059", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:41.680 INFO:teuthology.orchestra.run.smithi059.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:21:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:41 smithi059 ceph-mon[21819]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:41 smithi059 ceph-mon[26536]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:41 smithi064 ceph-mon[65271]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:21:42.321 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-26T16:21:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:21:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-04-26T16:21:42.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:42.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:42.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:42.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:42.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:42.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:42.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:42 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:42.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[21819]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:42.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:21:42.951 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:42.951 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:42 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:21:43.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:43 smithi059 ceph-mon[21819]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:43.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:43 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:44.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:43 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:44.273 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:21:44] "GET /metrics HTTP/1.1" 200 38073 "" "Prometheus/2.43.0" 2024-04-26T16:21:44.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:44.388+0000 7f470d9c5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:44.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:21:44.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:44.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:44.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:44.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:44.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:44.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:44.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:44.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:44.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:44.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:21:44.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:44.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:44.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:44.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.991 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-mon[21819]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 546 B/s rd, 0 op/s 2024-04-26T16:21:44.991 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-mon[21819]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:44.991 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:44.991 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:44.991 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:44.991 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:44.991 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.991 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.991 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.992 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:44.993 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:44.994 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:44.995 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:44.995 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:44.995 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:44.995 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-mon[26536]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 546 B/s rd, 0 op/s 2024-04-26T16:21:44.995 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:44.995 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:44.995 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:44.995 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:44.995 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:44.995 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:44.996 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:21:44.997 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:44.998 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:44.999 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-mon[21819]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:21:45.000 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:45.000 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:45.000 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:45.000 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:45.000 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:45.000 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:44 smithi059 ceph-mon[26536]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:21:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:44 smithi064 ceph-mon[65271]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 546 B/s rd, 0 op/s 2024-04-26T16:21:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:44 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:45.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:21:45.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:21:45.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:21:45.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:21:45.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:21:45.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:21:45.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:21:45.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:21:45.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:21:45.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:21:45.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:44 smithi064 ceph-mon[65271]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:21:45.465 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:21:45.737 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:45 smithi059 ceph-mon[26536]: from='client.25591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:45.737 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:45 smithi059 ceph-mon[21819]: from='client.25591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:45 smithi064 ceph-mon[65271]: from='client.25591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (2m) 2m ago 24m 16.7M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (114s) 40s ago 24m 77.5M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 2m ago 23m 61.2M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (3m) 40s ago 28m 466M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (13m) 2m ago 30m 518M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (30m) 2m ago 30m 79.3M 2048M 17.2.0 e1d6a67b021e 052d9cc2e6db 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (45s) 40s ago 29m 22.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (29m) 2m ago 29m 59.6M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (3m) 2m ago 25m 8493k - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:21:46.192 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (3m) 40s ago 25m 16.9M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:21:46.193 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (28m) 2m ago 28m 54.3M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:21:46.193 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (28m) 2m ago 28m 54.2M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:21:46.193 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (27m) 2m ago 27m 52.1M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:21:46.193 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (27m) 2m ago 27m 52.5M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:21:46.193 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (27m) 40s ago 27m 55.3M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:21:46.193 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (26m) 40s ago 26m 54.4M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:21:46.193 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (26m) 40s ago 26m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:21:46.194 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (26m) 40s ago 25m 53.4M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:21:46.194 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (2m) 40s ago 25m 62.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:21:46.194 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (23m) 2m ago 23m 75.8M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:21:46.194 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (23m) 40s ago 23m 76.3M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:21:46.621 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:21:46.621 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:21:46.621 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-26T16:21:46.621 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T16:21:46.621 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:21:46.621 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:21:46.621 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:21:46.621 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": {}, 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:21:46.622 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:21:46.716 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:46 smithi059 ceph-mon[21819]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:46.716 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:46 smithi059 ceph-mon[21819]: from='client.15681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:46.716 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:46 smithi059 ceph-mon[21819]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:46.717 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:46 smithi059 ceph-mon[21819]: from='client.? 172.21.15.59:0/4217127356' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:46 smithi059 ceph-mon[26536]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:46 smithi059 ceph-mon[26536]: from='client.15681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:46 smithi059 ceph-mon[26536]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:46 smithi059 ceph-mon[26536]: from='client.? 172.21.15.59:0/4217127356' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:46 smithi064 ceph-mon[65271]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:46 smithi064 ceph-mon[65271]: from='client.15681 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:46 smithi064 ceph-mon[65271]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:46 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/4217127356' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:46.990 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:21:46.990 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:21:46.990 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:21:46.990 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi059", 2024-04-26T16:21:46.990 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:21:46.990 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "0/2 daemons upgraded", 2024-04-26T16:21:46.990 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading mon daemons", 2024-04-26T16:21:46.990 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:21:46.990 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:21:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:21:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:21:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: Upgrade: Setting container_image for all mgr 2024-04-26T16:21:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T16:21:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T16:21:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: Upgrade: It appears safe to stop mon.a 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[21819]: from='client.25615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: Upgrade: Setting container_image for all mgr 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: Upgrade: It appears safe to stop mon.a 2024-04-26T16:21:48.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:47 smithi059 ceph-mon[26536]: from='client.25615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mgr 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: Upgrade: It appears safe to stop mon.a 2024-04-26T16:21:48.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:47 smithi064 ceph-mon[65271]: from='client.25615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:21:49.073 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[26536]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:49.074 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:49.074 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[26536]: Upgrade: Updating mon.a 2024-04-26T16:21:49.074 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:49.074 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:21:49.074 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:21:49.074 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[26536]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:49.074 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[26536]: Deploying daemon mon.a on smithi059 2024-04-26T16:21:49.074 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[21819]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:49.074 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:49.074 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[21819]: Upgrade: Updating mon.a 2024-04-26T16:21:49.075 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[21819]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:49.075 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:21:49.075 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:21:49.075 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[21819]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:49.075 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:48 smithi059 ceph-mon[21819]: Deploying daemon mon.a on smithi059 2024-04-26T16:21:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:48 smithi064 ceph-mon[65271]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:48 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:21:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:48 smithi064 ceph-mon[65271]: Upgrade: Updating mon.a 2024-04-26T16:21:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:48 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='mgr.y' 2024-04-26T16:21:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:48 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:21:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:48 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:21:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:48 smithi064 ceph-mon[65271]: from='mgr.25240 172.21.15.59:0/2741424399' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:21:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:48 smithi064 ceph-mon[65271]: Deploying daemon mon.a on smithi059 2024-04-26T16:21:50.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:50 smithi059 systemd[1]: Stopping Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:21:50.813 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a[21815]: 2024-04-26T16:21:50.420+0000 7f870615d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:21:50.814 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a[21815]: 2024-04-26T16:21:50.420+0000 7f870615d700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-26T16:21:50.814 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:50 smithi059 podman[80987]: 2024-04-26 16:21:50.482716372 +0000 UTC m=+0.164945121 container died 052d9cc2e6db782849eb0f4822c9d92e2fc8b7388d29dbed0216ebc8a49998e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, vcs-type=git, maintainer=Guillaume Abrioux , release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_BRANCH=HEAD, architecture=x86_64, io.buildah.version=1.19.8, version=8) 2024-04-26T16:21:51.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:50 smithi059 podman[80987]: 2024-04-26 16:21:50.904407902 +0000 UTC m=+0.586636650 container cleanup 052d9cc2e6db782849eb0f4822c9d92e2fc8b7388d29dbed0216ebc8a49998e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, ceph=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, release=754, io.buildah.version=1.19.8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.openshift.tags=base centos centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD) 2024-04-26T16:21:51.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:50 smithi059 bash[80987]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a 2024-04-26T16:21:51.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:51 smithi059 podman[81001]: 2024-04-26 16:21:51.164134323 +0000 UTC m=+0.680415994 container remove 052d9cc2e6db782849eb0f4822c9d92e2fc8b7388d29dbed0216ebc8a49998e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.component=centos-stream-container, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, ceph=True, maintainer=Guillaume Abrioux , version=8, GIT_CLEAN=True, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, RELEASE=HEAD) 2024-04-26T16:21:51.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:51 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T16:21:51.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:51 smithi059 systemd[1]: Stopped Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:21:51.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:51 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a.service: Consumed 28.595s CPU time. 2024-04-26T16:21:51.933 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:51 smithi059 systemd[1]: Starting Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:21:51.933 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:51 smithi059 podman[81132]: 2024-04-26 16:21:51.807139072 +0000 UTC m=+0.115586304 container create a406f6807e94bb9bf5d911c5a2c8b32003c35d518bef652f1ff16aa50ed98b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True) 2024-04-26T16:21:51.933 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:51 smithi059 podman[81132]: 2024-04-26 16:21:51.710476478 +0000 UTC m=+0.018923696 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:21:52.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 podman[81132]: 2024-04-26 16:21:52.059899953 +0000 UTC m=+0.368347149 container init a406f6807e94bb9bf5d911c5a2c8b32003c35d518bef652f1ff16aa50ed98b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:21:52.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 podman[81132]: 2024-04-26 16:21:52.063399007 +0000 UTC m=+0.371846208 container start a406f6807e94bb9bf5d911c5a2c8b32003c35d518bef652f1ff16aa50ed98b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-26T16:21:52.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T16:21:52.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev), process ceph-mon, pid 2 2024-04-26T16:21:52.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: pidfile_write: ignore empty --pid-file 2024-04-26T16:21:52.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: load: jerasure load: lrc 2024-04-26T16:21:52.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: RocksDB version: 7.9.2 2024-04-26T16:21:52.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Git sha 0 2024-04-26T16:21:52.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Compile date 2024-04-26 03:22:31 2024-04-26T16:21:52.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: DB SUMMARY 2024-04-26T16:21:52.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: DB Session ID: OWLA2I86RK9EIMM3T66Y 2024-04-26T16:21:52.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: CURRENT file: CURRENT 2024-04-26T16:21:52.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: MANIFEST file: MANIFEST-000015 size: 3102 Bytes 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000063.sst 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000061.log size: 564558 ; 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.error_if_exists: 0 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.create_if_missing: 0 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.paranoid_checks: 1 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.env: 0x55bdee2a4cc0 2024-04-26T16:21:52.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.info_log: 0x55bdef22dd80 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.statistics: (nil) 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.use_fsync: 0 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_log_file_size: 0 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.allow_fallocate: 1 2024-04-26T16:21:52.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T16:21:52.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T16:21:52.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.use_direct_reads: 0 2024-04-26T16:21:52.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T16:21:52.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T16:21:52.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.db_log_dir: 2024-04-26T16:21:52.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.wal_dir: 2024-04-26T16:21:52.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T16:21:52.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T16:21:52.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T16:21:52.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T16:21:52.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T16:21:52.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T16:21:52.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T16:21:52.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T16:21:52.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.write_buffer_manager: 0x55bdef1ff0e0 2024-04-26T16:21:52.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T16:21:52.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T16:21:52.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T16:21:52.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T16:21:52.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T16:21:52.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T16:21:52.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T16:21:52.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T16:21:52.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.unordered_write: 0 2024-04-26T16:21:52.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T16:21:52.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T16:21:52.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T16:21:52.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T16:21:52.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.row_cache: None 2024-04-26T16:21:52.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.wal_filter: None 2024-04-26T16:21:52.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T16:21:52.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.two_write_queues: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.wal_compression: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.atomic_flush: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.log_readahead_size: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T16:21:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T16:21:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T16:21:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_background_jobs: 2 2024-04-26T16:21:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_background_compactions: -1 2024-04-26T16:21:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_subcompactions: 1 2024-04-26T16:21:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T16:21:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T16:21:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T16:21:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_open_files: -1 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_background_flushes: -1 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Compression algorithms supported: 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: kZSTD supported: 0 2024-04-26T16:21:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: kXpressCompression supported: 0 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: kZlibCompression supported: 1 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: kSnappyCompression supported: 1 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: kLZ4Compression supported: 1 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: kBZip2Compression supported: 0 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.merge_operator: 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_filter: None 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T16:21:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bdef22df40) 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: cache_index_and_filter_blocks: 1 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: pin_top_level_index_and_filter: 1 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: index_type: 0 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: data_block_index_type: 0 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: index_shortening: 1 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: checksum: 4 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: no_block_cache: 0 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_cache: 0x55bdef2311f0 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_cache_name: BinnedLRUCache 2024-04-26T16:21:52.345 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_cache_options: 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: capacity : 536870912 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: num_shard_bits : 4 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: strict_capacity_limit : 0 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: high_pri_pool_ratio: 0.000 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_cache_compressed: (nil) 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: persistent_cache: (nil) 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_size: 4096 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_size_deviation: 10 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_restart_interval: 16 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: index_block_restart_interval: 1 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: metadata_block_size: 4096 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: partition_filters: 0 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_delta_encoding: 1 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: filter_policy: bloomfilter 2024-04-26T16:21:52.346 INFO:journalctl@ceph.mon.a.smithi059.stdout: whole_key_filtering: 1 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout: verify_compression: 0 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout: read_amp_bytes_per_bit: 0 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout: format_version: 5 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout: enable_index_compression: 1 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout: block_align: 0 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout: max_auto_readahead_size: 262144 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout: prepopulate_block_cache: 0 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout: initial_auto_readahead_size: 8192 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compression: NoCompression 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T16:21:52.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.num_levels: 7 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T16:21:52.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T16:21:52.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T16:21:52.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.inplace_update_support: 0 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T16:21:52.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.bloom_locality: 0 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.max_successive_merges: 0 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.ttl: 2592000 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T16:21:52.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.enable_blob_files: false 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.min_blob_size: 0 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 63.sst 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 65, last_sequence is 29327, log_number is 61,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 61 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 419b6a46-82f7-42e5-9231-597832c6caef 2024-04-26T16:21:52.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714148512093653, "job": 1, "event": "recovery_started", "wal_files": [61]} 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #61 mode 2 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714148512096543, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 66, "file_size": 552724, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 29328, "largest_seqno": 29829, "table_properties": {"data_size": 549764, "index_size": 1344, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 5068, "raw_average_key_size": 23, "raw_value_size": 545135, "raw_average_value_size": 2500, "num_data_blocks": 60, "num_entries": 218, "num_filter_entries": 218, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714148512, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "419b6a46-82f7-42e5-9231-597832c6caef", "db_session_id": "OWLA2I86RK9EIMM3T66Y", "orig_file_number": 66, "seqno_to_time_mapping": "N/A"}} 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714148512096631, "job": 1, "event": "recovery_finished"} 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/version_set.cc:5047] Creating manifest 68 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000061.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55bdef34e000 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: DB pointer 0x55bdef33a000 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** DB Stats ** 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T16:21:52.354 INFO:journalctl@ceph.mon.a.smithi059.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: L0 1/0 539.77 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 286.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: L6 1/0 9.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: Sum 2/0 10.46 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 286.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 286.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T16:21:52.355 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 286.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T16:21:52.356 INFO:journalctl@ceph.mon.a.smithi059.stdout: Cumulative compaction: 0.00 GB write, 101.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: Interval compaction: 0.00 GB write, 101.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: Block cache BinnedLRUCache@0x55bdef2311f0#2 capacity: 512.00 MB usage: 2.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.66 KB,0.00012517%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: starting mon.a rank 0 at public addrs [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] at bind addrs [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: mon.a@-1(???) e3 preinit fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: mon.a@-1(???).mds e1 new map 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: mon.a@-1(???).mds e1 print_map 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: e1 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: legacy client fscid: -1 2024-04-26T16:21:52.357 INFO:journalctl@ceph.mon.a.smithi059.stdout: 2024-04-26T16:21:52.358 INFO:journalctl@ceph.mon.a.smithi059.stdout: No filesystems configured 2024-04-26T16:21:52.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: mon.a@-1(???).osd e106 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T16:21:52.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: mon.a@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T16:21:52.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: mon.a@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T16:21:52.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: mon.a@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T16:21:52.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 ceph-mon[81151]: mon.a@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-04-26T16:21:52.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 bash[81132]: a406f6807e94bb9bf5d911c5a2c8b32003c35d518bef652f1ff16aa50ed98b61 2024-04-26T16:21:52.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:52 smithi059 systemd[1]: Started Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:21:53.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ignoring --setuser ceph since I am not root 2024-04-26T16:21:53.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ignoring --setgroup ceph since I am not root 2024-04-26T16:21:53.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mgr[60319]: -- 172.21.15.59:0/3294908221 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558c47a8e5a0 con 0x558c47a62800 2024-04-26T16:21:53.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:53.723+0000 7fe312aa1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: mon.a calling monitor election 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: monmap epoch 3 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: last_changed 2024-04-26T15:52:32.680547+0000 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: created 2024-04-26T15:50:59.741700+0000 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: min_mon_release 17 (quincy) 2024-04-26T16:21:53.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: election_strategy: 1 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: fsmap 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: mgrmap e42: y(active, since 4m), standbys: x 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: from='mgr.25240 ' entity='' 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[81151]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-26T16:21:53.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: mon.a calling monitor election 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: monmap epoch 3 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: last_changed 2024-04-26T15:52:32.680547+0000 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: created 2024-04-26T15:50:59.741700+0000 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: min_mon_release 17 (quincy) 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: election_strategy: 1 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:21:53.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: fsmap 2024-04-26T16:21:53.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T16:21:53.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: mgrmap e42: y(active, since 4m), standbys: x 2024-04-26T16:21:53.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:21:53.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:21:53.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:21:53.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: from='mgr.25240 ' entity='' 2024-04-26T16:21:53.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-mon[26536]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-26T16:21:53.911 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: ignoring --setuser ceph since I am not root 2024-04-26T16:21:53.911 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: ignoring --setgroup ceph since I am not root 2024-04-26T16:21:53.912 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mgr[58194]: -- 172.21.15.64:0/1603423657 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cbcb3dc5a0 con 0x55cbcb3b0800 2024-04-26T16:21:53.912 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:53.716+0000 7f47893b1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T16:21:53.912 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:53.907+0000 7f47893b1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:21:53.912 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:21:53.912 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:21:53.912 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:21:53.912 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: mon.a calling monitor election 2024-04-26T16:21:53.913 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T16:21:53.913 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: monmap epoch 3 2024-04-26T16:21:53.913 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:21:53.913 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: last_changed 2024-04-26T15:52:32.680547+0000 2024-04-26T16:21:53.913 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: created 2024-04-26T15:50:59.741700+0000 2024-04-26T16:21:53.913 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: min_mon_release 17 (quincy) 2024-04-26T16:21:53.913 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: election_strategy: 1 2024-04-26T16:21:53.913 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:21:53.913 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:21:53.913 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:21:53.914 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: fsmap 2024-04-26T16:21:53.914 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T16:21:53.914 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: mgrmap e42: y(active, since 4m), standbys: x 2024-04-26T16:21:53.914 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:21:53.914 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:21:53.914 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:21:53.914 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: from='mgr.25240 ' entity='' 2024-04-26T16:21:53.914 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:53 smithi064 ceph-mon[65271]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-26T16:21:54.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:54 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:54.015+0000 7f47893b1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T16:21:54.241 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:53.914+0000 7fe312aa1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:21:54.242 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:54.021+0000 7fe312aa1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T16:21:54.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:54.342+0000 7fe312aa1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T16:21:54.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:54.430+0000 7fe312aa1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T16:21:54.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:21:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:21:54.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:21:54.583 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:21:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:21:54.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:21:54.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:54 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:54.337+0000 7f47893b1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T16:21:54.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:54 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:54.426+0000 7f47893b1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T16:21:55.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:21:55 smithi059 ceph-mon[81151]: mgrmap e44: y(active, since 4m), standbys: x 2024-04-26T16:21:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:21:55 smithi059 ceph-mon[26536]: mgrmap e44: y(active, since 4m), standbys: x 2024-04-26T16:21:55.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:55.690+0000 7fe312aa1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T16:21:55.872 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:21:55 smithi064 ceph-mon[65271]: mgrmap e44: y(active, since 4m), standbys: x 2024-04-26T16:21:55.873 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:55.694+0000 7f47893b1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T16:21:56.186 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:55.872+0000 7f47893b1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T16:21:56.187 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:56 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:56.057+0000 7f47893b1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T16:21:56.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:55 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:55.869+0000 7fe312aa1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T16:21:56.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:56.049+0000 7fe312aa1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T16:21:56.664 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:56.352+0000 7fe312aa1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T16:21:56.665 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:56.450+0000 7fe312aa1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T16:21:56.674 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:56 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:56.362+0000 7f47893b1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T16:21:56.674 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:56 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:56.460+0000 7f47893b1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T16:21:56.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:56 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:56.674+0000 7f47893b1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T16:21:56.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:56 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:56.877+0000 7f47893b1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:21:57.082 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:56.664+0000 7fe312aa1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T16:21:57.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:56.867+0000 7fe312aa1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T16:21:57.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:57.511+0000 7fe312aa1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T16:21:57.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:57.745+0000 7fe312aa1200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-04-26T16:21:57.936 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:57 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:57.514+0000 7f47893b1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T16:21:57.937 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:57 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:57.749+0000 7f47893b1200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-04-26T16:21:58.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:58 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:58.195+0000 7fe312aa1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T16:21:58.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:58 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:58.297+0000 7fe312aa1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T16:21:58.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:58.198+0000 7f47893b1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T16:21:58.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:58.301+0000 7f47893b1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T16:21:59.067 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:58 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:58.689+0000 7fe312aa1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T16:21:59.070 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:58.691+0000 7f47893b1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T16:21:59.070 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:59 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:59.070+0000 7f47893b1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T16:21:59.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:59.067+0000 7fe312aa1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T16:21:59.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:59.166+0000 7fe312aa1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T16:21:59.430 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:59 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:59.169+0000 7f47893b1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T16:21:59.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:59 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:59.429+0000 7f47893b1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T16:21:59.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:59 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:59.572+0000 7f47893b1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T16:21:59.687 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:21:59 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:21:59.674+0000 7f47893b1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T16:21:59.832 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:59.428+0000 7fe312aa1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T16:21:59.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:59.570+0000 7fe312aa1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T16:21:59.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:21:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:21:59.672+0000 7fe312aa1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T16:22:00.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:22:00.016+0000 7fe312aa1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T16:22:00.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:22:00.115+0000 7fe312aa1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T16:22:00.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:22:00 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:22:00.014+0000 7f47893b1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T16:22:00.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:22:00 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:22:00.113+0000 7f47893b1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T16:22:01.046 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:22:00.755+0000 7fe312aa1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T16:22:01.046 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:22:00.989+0000 7fe312aa1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T16:22:01.047 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:22:00 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:22:00.754+0000 7f47893b1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T16:22:01.047 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:22:00 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:22:00.989+0000 7f47893b1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T16:22:01.047 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:22:00 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: [26/Apr/2024:16:22:00] ENGINE Bus STARTING 2024-04-26T16:22:01.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:22:01] ENGINE Bus STARTING 2024-04-26T16:22:01.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CherryPy Checker: 2024-04-26T16:22:01.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: The Application mounted at '' has an empty config. 2024-04-26T16:22:01.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:22:01.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:22:01] ENGINE Serving on http://:::9283 2024-04-26T16:22:01.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:22:01] ENGINE Bus STARTED 2024-04-26T16:22:01.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: Active manager daemon y restarted 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: Activating manager daemon y 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: Standby manager daemon x restarted 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: Standby manager daemon x started 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: mgrmap e45: y(active, starting, since 0.00670197s), standbys: x 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:22:01.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:22:01.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[26536]: Manager daemon y is now available 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: Active manager daemon y restarted 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: Activating manager daemon y 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: Standby manager daemon x restarted 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: Standby manager daemon x started 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: mgrmap e45: y(active, starting, since 0.00670197s), standbys: x 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:22:01.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:22:01.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-mon[81151]: Manager daemon y is now available 2024-04-26T16:22:01.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: CherryPy Checker: 2024-04-26T16:22:01.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: The Application mounted at '' has an empty config. 2024-04-26T16:22:01.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: 2024-04-26T16:22:01.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: [26/Apr/2024:16:22:01] ENGINE Serving on http://:::9283 2024-04-26T16:22:01.437 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x[58190]: [26/Apr/2024:16:22:01] ENGINE Bus STARTED 2024-04-26T16:22:01.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: Active manager daemon y restarted 2024-04-26T16:22:01.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: Activating manager daemon y 2024-04-26T16:22:01.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T16:22:01.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: Standby manager daemon x restarted 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: Standby manager daemon x started 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: mgrmap e45: y(active, starting, since 0.00670197s), standbys: x 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.? 172.21.15.64:0/2609694078' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T16:22:01.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T16:22:01.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:22:01.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:22:01.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:22:01.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:22:01.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:22:01.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:22:01.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:22:01.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:22:01.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:01 smithi064 ceph-mon[65271]: Manager daemon y is now available 2024-04-26T16:22:02.011 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:22:01 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:22:01.746Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=7 2024-04-26T16:22:02.264 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:22:02.007+0000 7fe2d86fd700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-26T16:22:02.265 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:22:02.265 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:02.265 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:22:02.265 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:22:02.265 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-mon[81151]: mgrmap e46: y(active, since 1.01153s), standbys: x 2024-04-26T16:22:02.265 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:22:02.265 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:02.265 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:22:02.266 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:22:02.266 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:02 smithi059 ceph-mon[26536]: mgrmap e46: y(active, since 1.01153s), standbys: x 2024-04-26T16:22:02.318 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:22:02.318 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:02.319 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T16:22:02.319 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T16:22:02.319 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:02 smithi064 ceph-mon[65271]: mgrmap e46: y(active, since 1.01153s), standbys: x 2024-04-26T16:22:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:03 smithi064 ceph-mon[65271]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:03 smithi064 ceph-mon[65271]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T16:22:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:03 smithi064 ceph-mon[65271]: Cluster is now healthy 2024-04-26T16:22:03.332 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:22:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:22:03.089Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=7 2024-04-26T16:22:03.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:03 smithi059 ceph-mon[81151]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:03.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:03 smithi059 ceph-mon[81151]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T16:22:03.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:03 smithi059 ceph-mon[81151]: Cluster is now healthy 2024-04-26T16:22:03.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:03 smithi059 ceph-mon[26536]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:03.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:03 smithi059 ceph-mon[26536]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T16:22:03.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:03 smithi059 ceph-mon[26536]: Cluster is now healthy 2024-04-26T16:22:04.314 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:22:04] "GET /metrics HTTP/1.1" 200 35278 "" "Prometheus/2.43.0" 2024-04-26T16:22:04.315 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[81151]: [26/Apr/2024:16:22:02] ENGINE Bus STARTING 2024-04-26T16:22:04.315 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:04.315 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[81151]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:04.315 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[81151]: mgrmap e47: y(active, since 2s), standbys: x 2024-04-26T16:22:04.315 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[81151]: [26/Apr/2024:16:22:03] ENGINE Serving on http://172.21.15.59:8765 2024-04-26T16:22:04.315 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[81151]: [26/Apr/2024:16:22:03] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T16:22:04.315 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[81151]: [26/Apr/2024:16:22:03] ENGINE Bus STARTED 2024-04-26T16:22:04.316 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[26536]: [26/Apr/2024:16:22:02] ENGINE Bus STARTING 2024-04-26T16:22:04.316 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:04.316 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[26536]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:04.316 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[26536]: mgrmap e47: y(active, since 2s), standbys: x 2024-04-26T16:22:04.316 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[26536]: [26/Apr/2024:16:22:03] ENGINE Serving on http://172.21.15.59:8765 2024-04-26T16:22:04.316 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[26536]: [26/Apr/2024:16:22:03] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T16:22:04.316 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:04 smithi059 ceph-mon[26536]: [26/Apr/2024:16:22:03] ENGINE Bus STARTED 2024-04-26T16:22:04.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:04 smithi064 ceph-mon[65271]: [26/Apr/2024:16:22:02] ENGINE Bus STARTING 2024-04-26T16:22:04.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:04 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:04.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:04 smithi064 ceph-mon[65271]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:04.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:04 smithi064 ceph-mon[65271]: mgrmap e47: y(active, since 2s), standbys: x 2024-04-26T16:22:04.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:04 smithi064 ceph-mon[65271]: [26/Apr/2024:16:22:03] ENGINE Serving on http://172.21.15.59:8765 2024-04-26T16:22:04.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:04 smithi064 ceph-mon[65271]: [26/Apr/2024:16:22:03] ENGINE Serving on https://172.21.15.59:7150 2024-04-26T16:22:04.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:04 smithi064 ceph-mon[65271]: [26/Apr/2024:16:22:03] ENGINE Bus STARTED 2024-04-26T16:22:06.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[81151]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[81151]: mgrmap e48: y(active, since 4s), standbys: x 2024-04-26T16:22:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[26536]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:06.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[26536]: mgrmap e48: y(active, since 4s), standbys: x 2024-04-26T16:22:06.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:06 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:06 smithi064 ceph-mon[65271]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:06.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:06 smithi064 ceph-mon[65271]: mgrmap e48: y(active, since 4s), standbys: x 2024-04-26T16:22:06.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:06 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:06 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:06 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:06.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:06 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.177 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.177 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.177 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:07 smithi059 ceph-mon[81151]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:08.177 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.177 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.177 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:07 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.177 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:07 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.177 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:07 smithi059 ceph-mon[26536]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:08.178 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:07 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.178 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:07 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:07 smithi064 ceph-mon[65271]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:22:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:08.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:10.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:10 smithi064 ceph-mon[65271]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:22:10.329 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:10 smithi059 ceph-mon[26536]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:22:10.329 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:10 smithi059 ceph-mon[81151]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T16:22:11.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:11 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:22:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:22:12.537 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: Detected new or changed devices on smithi059 2024-04-26T16:22:12.537 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:22:12.537 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:22:12.537 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:22:12.537 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:22:12.537 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:22:12.537 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:22:12.537 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:22:12.538 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:22:12.538 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:22:12.538 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.538 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.538 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.538 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.538 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.538 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:22:12.538 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:12.538 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: Detected new or changed devices on smithi059 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:22:12.540 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: Detected new or changed devices on smithi059 2024-04-26T16:22:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: Updating smithi059:/etc/ceph/ceph.conf 2024-04-26T16:22:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: Updating smithi064:/etc/ceph/ceph.conf 2024-04-26T16:22:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:22:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T16:22:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T16:22:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:22:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: Updating smithi059:/etc/ceph/ceph.client.admin.keyring 2024-04-26T16:22:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: Updating smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:22:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: Updating smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T16:22:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:22:12.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:13.482 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-mon[26536]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:22:13.483 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-mon[26536]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:22:13.483 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-mon[26536]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:22:13.483 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-mon[26536]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:22:13.483 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-mon[81151]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:22:13.483 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:22:13.483 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:22:13.483 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-mon[81151]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:22:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:13 smithi064 ceph-mon[65271]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T16:22:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:13 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:22:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:13 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:22:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:13 smithi064 ceph-mon[65271]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:22:13.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:22:13.634+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:13.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:22:13.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:13.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:13.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:13.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:13.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:13.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:13.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:13.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:22:13.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:13.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:13.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:13.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:13.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:13.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:13.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:14.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:22:14] "GET /metrics HTTP/1.1" 200 35278 "" "Prometheus/2.43.0" 2024-04-26T16:22:14.483 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:14.483 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:22:14.483 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:22:14.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T16:22:14.484 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T16:22:14.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-mon[26536]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:14.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:22:14.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:22:14.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T16:22:14.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:14 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T16:22:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:14 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:22:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:22:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T16:22:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[26536]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:22:15.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:22:15.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:15.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:15.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:22:15.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:15.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:15.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:15.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:15.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:15.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:15.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:15.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:15.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:22:15.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[26536]: Upgrade: It appears safe to stop mon.c 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[26536]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[81151]: Upgrade: It appears safe to stop mon.c 2024-04-26T16:22:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[81151]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:22:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:15 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:15 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:22:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:22:15.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:15.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:15.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:15 smithi064 ceph-mon[65271]: Upgrade: It appears safe to stop mon.c 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:15 smithi064 ceph-mon[65271]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:15 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:15 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:15 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:15.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:15 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:16.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:16 smithi059 ceph-mon[81151]: Upgrade: Updating mon.c 2024-04-26T16:22:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:16 smithi059 ceph-mon[81151]: Deploying daemon mon.c on smithi059 2024-04-26T16:22:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:16 smithi059 ceph-mon[26536]: Upgrade: Updating mon.c 2024-04-26T16:22:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:16 smithi059 ceph-mon[26536]: Deploying daemon mon.c on smithi059 2024-04-26T16:22:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:16 smithi059 ceph-mon[26536]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:16 smithi064 ceph-mon[65271]: Upgrade: Updating mon.c 2024-04-26T16:22:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:16 smithi064 ceph-mon[65271]: Deploying daemon mon.c on smithi059 2024-04-26T16:22:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:17.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:17 smithi059 ceph-mon[81151]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:22:17.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:17 smithi059 systemd[1]: Stopping Ceph mon.c for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:22:17.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:17 smithi059 ceph-mon[26536]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:22:17.396 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:22:17.661 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c[26532]: 2024-04-26T16:22:17.335+0000 7ff31d18c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:22:17.661 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c[26532]: 2024-04-26T16:22:17.335+0000 7ff31d18c700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-26T16:22:17.661 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:17 smithi059 podman[86094]: 2024-04-26 16:22:17.358073841 +0000 UTC m=+0.147499492 container died 17313040ebe2f77c28d4531a3a9e89a1fa9acef98a9f00e4424d1ceba67fe2a4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, architecture=x86_64, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, distribution-scope=public, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, release=754, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, name=centos-stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-26T16:22:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:17 smithi064 ceph-mon[65271]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T16:22:17.961 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:17 smithi059 podman[86094]: 2024-04-26 16:22:17.752477004 +0000 UTC m=+0.541902645 container cleanup 17313040ebe2f77c28d4531a3a9e89a1fa9acef98a9f00e4424d1ceba67fe2a4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, release=754, name=centos-stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, vcs-type=git, GIT_CLEAN=True, RELEASE=HEAD) 2024-04-26T16:22:17.961 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:17 smithi059 bash[86094]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c 2024-04-26T16:22:18.132 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:22:18.132 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (2m) 12s ago 25m 17.2M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:22:18.132 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (2m) 12s ago 24m 80.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:22:18.132 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 12s ago 24m 61.9M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:22:18.132 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (4m) 12s ago 29m 461M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:22:18.132 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (14m) 12s ago 31m 486M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:22:18.132 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (26s) 12s ago 31m 32.4M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:22:18.132 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (77s) 12s ago 29m 40.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:22:18.132 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (29m) 12s ago 29m 67.2M 2048M 17.2.0 e1d6a67b021e 17313040ebe2 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (3m) 12s ago 25m 8312k - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (3m) 12s ago 25m 17.2M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (29m) 12s ago 29m 54.6M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (28m) 12s ago 28m 54.4M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (28m) 12s ago 28m 52.7M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (28m) 12s ago 28m 52.6M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (27m) 12s ago 27m 55.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (27m) 12s ago 27m 54.5M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (26m) 12s ago 26m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (26m) 12s ago 26m 53.6M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (3m) 12s ago 25m 66.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (24m) 12s ago 24m 77.0M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:22:18.133 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (24m) 12s ago 24m 77.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:22:18.322 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:17 smithi059 podman[86108]: 2024-04-26 16:22:17.962100747 +0000 UTC m=+0.600805527 container remove 17313040ebe2f77c28d4531a3a9e89a1fa9acef98a9f00e4424d1ceba67fe2a4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, name=centos-stream, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , release=754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, RELEASE=HEAD, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.display-name=CentOS Stream 8) 2024-04-26T16:22:18.322 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-26T16:22:18.323 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 systemd[1]: Stopped Ceph mon.c for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:22:18.323 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.c.service: Consumed 18.111s CPU time. 2024-04-26T16:22:18.323 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 systemd[1]: Starting Ceph mon.c for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:22:18.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 podman[86307]: 2024-04-26 16:22:18.451042792 +0000 UTC m=+0.018662004 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:22:18.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 podman[86307]: 2024-04-26 16:22:18.571912254 +0000 UTC m=+0.139531439 container create 0c13fa03c2b8252699e325dfa507d5aa1e19e97e9a84b4e49eac43b2e920f9a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:22:18.584 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:22:18.584 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:22:18.584 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:22:18.585 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-04-26T16:22:18.586 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 4 2024-04-26T16:22:18.586 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:22:18.586 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:22:18.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 podman[86307]: 2024-04-26 16:22:18.799886857 +0000 UTC m=+0.367506044 container init 0c13fa03c2b8252699e325dfa507d5aa1e19e97e9a84b4e49eac43b2e920f9a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:22:18.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 podman[86307]: 2024-04-26 16:22:18.803075771 +0000 UTC m=+0.370694959 container start 0c13fa03c2b8252699e325dfa507d5aa1e19e97e9a84b4e49eac43b2e920f9a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:22:18.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T16:22:18.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev), process ceph-mon, pid 2 2024-04-26T16:22:18.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: pidfile_write: ignore empty --pid-file 2024-04-26T16:22:18.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: load: jerasure load: lrc 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: RocksDB version: 7.9.2 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Git sha 0 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Compile date 2024-04-26 03:22:31 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: DB SUMMARY 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: DB Session ID: UZ55FODX9DRI1G8OJCYN 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: CURRENT file: CURRENT 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3217 Bytes 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 2551180 ; 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.error_if_exists: 0 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.create_if_missing: 0 2024-04-26T16:22:18.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.paranoid_checks: 1 2024-04-26T16:22:18.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T16:22:18.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T16:22:18.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T16:22:18.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.env: 0x564ff6ac2cc0 2024-04-26T16:22:18.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T16:22:18.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.info_log: 0x564ff8621d80 2024-04-26T16:22:18.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T16:22:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.statistics: (nil) 2024-04-26T16:22:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.use_fsync: 0 2024-04-26T16:22:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_log_file_size: 0 2024-04-26T16:22:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T16:22:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T16:22:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T16:22:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T16:22:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.allow_fallocate: 1 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.use_direct_reads: 0 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.db_log_dir: 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.wal_dir: 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T16:22:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.write_buffer_manager: 0x564ff85f30e0 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.unordered_write: 0 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T16:22:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.row_cache: None 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.wal_filter: None 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.two_write_queues: 0 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.wal_compression: 0 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.atomic_flush: 0 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.log_readahead_size: 0 2024-04-26T16:22:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_background_jobs: 2 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_background_compactions: -1 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_subcompactions: 1 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T16:22:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_open_files: -1 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_background_flushes: -1 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Compression algorithms supported: 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: kZSTD supported: 0 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: kXpressCompression supported: 0 2024-04-26T16:22:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: kZlibCompression supported: 1 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: kSnappyCompression supported: 1 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: kLZ4Compression supported: 1 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: kBZip2Compression supported: 0 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.merge_operator: 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_filter: None 2024-04-26T16:22:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564ff8621f40) 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: cache_index_and_filter_blocks: 1 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: pin_top_level_index_and_filter: 1 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: index_type: 0 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: data_block_index_type: 0 2024-04-26T16:22:18.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: index_shortening: 1 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: checksum: 4 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: no_block_cache: 0 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_cache: 0x564ff86251f0 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_cache_name: BinnedLRUCache 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_cache_options: 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: capacity : 536870912 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: num_shard_bits : 4 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: strict_capacity_limit : 0 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: high_pri_pool_ratio: 0.000 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_cache_compressed: (nil) 2024-04-26T16:22:18.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: persistent_cache: (nil) 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_size: 4096 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_size_deviation: 10 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_restart_interval: 16 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: index_block_restart_interval: 1 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: metadata_block_size: 4096 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: partition_filters: 0 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_delta_encoding: 1 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: filter_policy: bloomfilter 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: whole_key_filtering: 1 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: verify_compression: 0 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: read_amp_bytes_per_bit: 0 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: format_version: 5 2024-04-26T16:22:18.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: enable_index_compression: 1 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout: block_align: 0 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout: max_auto_readahead_size: 262144 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout: prepopulate_block_cache: 0 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout: initial_auto_readahead_size: 8192 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compression: NoCompression 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.num_levels: 7 2024-04-26T16:22:18.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T16:22:18.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T16:22:18.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T16:22:18.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T16:22:18.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T16:22:18.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T16:22:18.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T16:22:18.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T16:22:18.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T16:22:18.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T16:22:18.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T16:22:18.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T16:22:18.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T16:22:18.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T16:22:18.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T16:22:18.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T16:22:18.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T16:22:18.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-26T16:22:18.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.inplace_update_support: 0 2024-04-26T16:22:18.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T16:22:18.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T16:22:18.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T16:22:18.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T16:22:18.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.bloom_locality: 0 2024-04-26T16:22:18.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.max_successive_merges: 0 2024-04-26T16:22:18.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T16:22:18.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T16:22:18.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T16:22:18.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T16:22:18.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.ttl: 2592000 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.enable_blob_files: false 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.min_blob_size: 0 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T16:22:18.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-04-26T16:22:18.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T16:22:18.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32793, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T16:22:18.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-04-26T16:22:18.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a07ca0c8-7c12-4610-bb5f-baa8a60b3cc4 2024-04-26T16:22:18.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714148538833485, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-04-26T16:22:18.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-04-26T16:22:18.957 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:22:18.957 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:22:18.957 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:22:18.957 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi059", 2024-04-26T16:22:18.957 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:22:18.958 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "1/2 daemons upgraded", 2024-04-26T16:22:18.958 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading mon daemons", 2024-04-26T16:22:18.958 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:22:18.958 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:22:19.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714148538841807, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1617796, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32799, "largest_seqno": 33256, "table_properties": {"data_size": 1614640, "index_size": 1540, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 5187, "raw_average_key_size": 24, "raw_value_size": 1609738, "raw_average_value_size": 7452, "num_data_blocks": 69, "num_entries": 216, "num_filter_entries": 216, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714148538, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a07ca0c8-7c12-4610-bb5f-baa8a60b3cc4", "db_session_id": "UZ55FODX9DRI1G8OJCYN", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-04-26T16:22:19.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714148538841912, "job": 1, "event": "recovery_finished"} 2024-04-26T16:22:19.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-04-26T16:22:19.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T16:22:19.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T16:22:19.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564ff8742000 2024-04-26T16:22:19.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: DB pointer 0x564ff872e000 2024-04-26T16:22:19.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T16:22:19.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T16:22:19.334 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** DB Stats ** 2024-04-26T16:22:19.334 INFO:journalctl@ceph.mon.c.smithi059.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T16:22:19.334 INFO:journalctl@ceph.mon.c.smithi059.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: L0 1/0 1.54 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 403.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: L6 1/0 10.88 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Sum 2/0 12.42 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 403.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T16:22:19.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 403.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** Compaction Stats [default] ** 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 403.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T16:22:19.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout: Cumulative compaction: 0.00 GB write, 119.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout: Interval compaction: 0.00 GB write, 119.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout: Block cache BinnedLRUCache@0x564ff86251f0#2 capacity: 512.00 MB usage: 2.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.66 KB,0.00012517%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: starting mon.c rank 1 at public addrs [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] at bind addrs [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: mon.c@-1(???) e3 preinit fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: mon.c@-1(???).mds e1 new map 2024-04-26T16:22:19.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: mon.c@-1(???).mds e1 print_map 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: e1 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: legacy client fscid: -1 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: No filesystems configured 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: mon.c@-1(???).osd e107 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: mon.c@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: mon.c@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: mon.c@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 ceph-mon[86348]: mon.c@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 bash[86307]: 0c13fa03c2b8252699e325dfa507d5aa1e19e97e9a84b4e49eac43b2e920f9a4 2024-04-26T16:22:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:18 smithi059 systemd[1]: Started Ceph mon.c for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:22:24.140 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:22:23.852+0000 7fe2d86fd700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-26T16:22:24.141 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:22:24.141 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:22:24.141 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:22:24.141 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: fsmap 2024-04-26T16:22:24.141 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:24.141 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: mgrmap e48: y(active, since 17s), standbys: x 2024-04-26T16:22:24.141 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: mon.a calling monitor election 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: mon.c calling monitor election 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: monmap epoch 4 2024-04-26T16:22:24.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: last_changed 2024-04-26T16:22:18.923604+0000 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: created 2024-04-26T15:50:59.741700+0000 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: min_mon_release 19 (squid) 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: election_strategy: 1 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: fsmap 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: mgrmap e48: y(active, since 22s), standbys: x 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:24.143 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: fsmap 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: mgrmap e48: y(active, since 17s), standbys: x 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:22:24.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: mon.a calling monitor election 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: mon.c calling monitor election 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: monmap epoch 4 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: last_changed 2024-04-26T16:22:18.923604+0000 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: created 2024-04-26T15:50:59.741700+0000 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: min_mon_release 19 (squid) 2024-04-26T16:22:24.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: election_strategy: 1 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: fsmap 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: mgrmap e48: y(active, since 22s), standbys: x 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:24.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:22:24.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:22:24] "GET /metrics HTTP/1.1" 200 38082 "" "Prometheus/2.43.0" 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: mon.b calling monitor election 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: mon.c calling monitor election 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: mon.a calling monitor election 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: monmap epoch 4 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: last_changed 2024-04-26T16:22:18.923604+0000 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: created 2024-04-26T15:50:59.741700+0000 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: min_mon_release 19 (squid) 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: election_strategy: 1 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:22:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: fsmap 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[81151]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:25 smithi059 ceph-mon[81151]: mgrmap e48: y(active, since 23s), standbys: x 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:25 smithi059 ceph-mon[81151]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:25 smithi059 ceph-mon[81151]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:25 smithi059 ceph-mon[81151]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: mon.b calling monitor election 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: mon.c calling monitor election 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: mon.a calling monitor election 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T16:22:25.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: monmap epoch 4 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: last_changed 2024-04-26T16:22:18.923604+0000 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: created 2024-04-26T15:50:59.741700+0000 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: min_mon_release 19 (squid) 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: election_strategy: 1 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: fsmap 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: mgrmap e48: y(active, since 23s), standbys: x 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:22:25.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:24 smithi059 ceph-mon[86348]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:22:25.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:22:25.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:22:25.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:22:25.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: fsmap 2024-04-26T16:22:25.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: mgrmap e48: y(active, since 17s), standbys: x 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: mon.a calling monitor election 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: mon.c calling monitor election 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:25.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: monmap epoch 4 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: last_changed 2024-04-26T16:22:18.923604+0000 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: created 2024-04-26T15:50:59.741700+0000 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: min_mon_release 19 (squid) 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: election_strategy: 1 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: fsmap 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: mgrmap e48: y(active, since 22s), standbys: x 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:22:25.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: mon.b calling monitor election 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: mon.c calling monitor election 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: mon.a calling monitor election 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: monmap epoch 4 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: fsid b6607c98-03e4-11ef-bc93-c7b262605968 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: last_changed 2024-04-26T16:22:18.923604+0000 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: created 2024-04-26T15:50:59.741700+0000 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: min_mon_release 19 (squid) 2024-04-26T16:22:25.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: election_strategy: 1 2024-04-26T16:22:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2024-04-26T16:22:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:24 smithi064 ceph-mon[65271]: 1: [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon.c 2024-04-26T16:22:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:25 smithi064 ceph-mon[65271]: 2: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.b 2024-04-26T16:22:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:25 smithi064 ceph-mon[65271]: fsmap 2024-04-26T16:22:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:25 smithi064 ceph-mon[65271]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T16:22:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:25 smithi064 ceph-mon[65271]: mgrmap e48: y(active, since 23s), standbys: x 2024-04-26T16:22:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:25 smithi064 ceph-mon[65271]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:22:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:25 smithi064 ceph-mon[65271]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:22:25.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:25 smithi064 ceph-mon[65271]: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:22:27.321 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:27 smithi059 ceph-mon[86348]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:27.322 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:27 smithi059 ceph-mon[81151]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:27.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:27 smithi064 ceph-mon[65271]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:29.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:29 smithi059 ceph-mon[86348]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:22:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:29 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:29.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:29 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:29 smithi059 ceph-mon[81151]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:22:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:29 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:29.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:29 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:29.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:29 smithi064 ceph-mon[65271]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:22:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:29 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:29 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:30.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:30.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:31.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:31 smithi064 ceph-mon[65271]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:31.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:31 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:31.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:31 smithi059 ceph-mon[86348]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:31 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:31.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:31 smithi059 ceph-mon[81151]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:31.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:31 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:33 smithi064 ceph-mon[65271]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:33 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:33 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:33 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:33 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:22:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:33 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:33 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:33 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:33 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:33.700 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[81151]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[86348]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:33.701 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:33.702 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:33.702 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:22:33.702 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:33.702 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:33.702 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:33.702 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:33 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:34.244 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:22:34] "GET /metrics HTTP/1.1" 200 38091 "" "Prometheus/2.43.0" 2024-04-26T16:22:34.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-mon[86348]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:22:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-mon[86348]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-mon[86348]: Reconfiguring mon.a (monmap changed)... 2024-04-26T16:22:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-mon[86348]: Reconfiguring daemon mon.a on smithi059 2024-04-26T16:22:34.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-mon[81151]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:22:34.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-mon[81151]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:34.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:34.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-mon[81151]: Reconfiguring mon.a (monmap changed)... 2024-04-26T16:22:34.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:34 smithi059 ceph-mon[81151]: Reconfiguring daemon mon.a on smithi059 2024-04-26T16:22:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:34 smithi064 ceph-mon[65271]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:22:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:34 smithi064 ceph-mon[65271]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:34 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:34 smithi064 ceph-mon[65271]: Reconfiguring mon.a (monmap changed)... 2024-04-26T16:22:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:34 smithi064 ceph-mon[65271]: Reconfiguring daemon mon.a on smithi059 2024-04-26T16:22:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[86348]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T16:22:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:22:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:22:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[86348]: Reconfiguring daemon mgr.y on smithi059 2024-04-26T16:22:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[86348]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:36.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:36.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:36.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[81151]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T16:22:36.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:22:36.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:22:36.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:36.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[81151]: Reconfiguring daemon mgr.y on smithi059 2024-04-26T16:22:36.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:35 smithi059 ceph-mon[81151]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:36.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:36.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:36.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:35 smithi064 ceph-mon[65271]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T16:22:36.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:22:36.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:22:36.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:36.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:35 smithi064 ceph-mon[65271]: Reconfiguring daemon mgr.y on smithi059 2024-04-26T16:22:36.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:35 smithi064 ceph-mon[65271]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:38.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:38.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:38.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[81151]: Reconfiguring mon.c (monmap changed)... 2024-04-26T16:22:38.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:38.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:38.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:38.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[81151]: Reconfiguring daemon mon.c on smithi059 2024-04-26T16:22:38.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[81151]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 558 B/s rd, 0 op/s 2024-04-26T16:22:38.026 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:38.027 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:38.027 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[86348]: Reconfiguring mon.c (monmap changed)... 2024-04-26T16:22:38.027 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:38.027 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:38.027 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:38.027 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[86348]: Reconfiguring daemon mon.c on smithi059 2024-04-26T16:22:38.027 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:37 smithi059 ceph-mon[86348]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 558 B/s rd, 0 op/s 2024-04-26T16:22:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:37 smithi064 ceph-mon[65271]: Reconfiguring mon.c (monmap changed)... 2024-04-26T16:22:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:37 smithi064 ceph-mon[65271]: Reconfiguring daemon mon.c on smithi059 2024-04-26T16:22:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:37 smithi064 ceph-mon[65271]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 558 B/s rd, 0 op/s 2024-04-26T16:22:39.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[86348]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[86348]: Reconfiguring daemon osd.0 on smithi059 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[81151]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:39 smithi059 ceph-mon[81151]: Reconfiguring daemon osd.0 on smithi059 2024-04-26T16:22:39.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:39 smithi064 ceph-mon[65271]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T16:22:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T16:22:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:39 smithi064 ceph-mon[65271]: Reconfiguring daemon osd.0 on smithi059 2024-04-26T16:22:40.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:40 smithi059 ceph-mon[86348]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:40.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:40.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:40.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T16:22:40.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:40.728 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:40 smithi059 ceph-mon[81151]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:40.728 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:40.728 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:40.728 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T16:22:40.728 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:40 smithi064 ceph-mon[65271]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T16:22:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:41.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:41 smithi059 ceph-mon[81151]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T16:22:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:41 smithi059 ceph-mon[81151]: Reconfiguring daemon osd.1 on smithi059 2024-04-26T16:22:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:41 smithi059 ceph-mon[86348]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T16:22:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:41 smithi059 ceph-mon[86348]: Reconfiguring daemon osd.1 on smithi059 2024-04-26T16:22:41.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:41 smithi064 ceph-mon[65271]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T16:22:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:41 smithi064 ceph-mon[65271]: Reconfiguring daemon osd.1 on smithi059 2024-04-26T16:22:42.725 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:42 smithi059 ceph-mon[86348]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:42.726 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:42.726 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:42.726 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T16:22:42.726 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:42.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:42 smithi059 ceph-mon[81151]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:42.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:42.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:42.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T16:22:42.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:42 smithi064 ceph-mon[65271]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:22:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T16:22:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:43.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:43 smithi059 ceph-mon[81151]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T16:22:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:43 smithi059 ceph-mon[81151]: Reconfiguring daemon osd.2 on smithi059 2024-04-26T16:22:43.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:43 smithi059 ceph-mon[86348]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T16:22:43.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:43 smithi059 ceph-mon[86348]: Reconfiguring daemon osd.2 on smithi059 2024-04-26T16:22:43.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:43 smithi064 ceph-mon[65271]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T16:22:43.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:43 smithi064 ceph-mon[65271]: Reconfiguring daemon osd.2 on smithi059 2024-04-26T16:22:44.374 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:22:44] "GET /metrics HTTP/1.1" 200 38091 "" "Prometheus/2.43.0" 2024-04-26T16:22:44.661 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[81151]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:22:44.661 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:44.661 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:44.661 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:44.661 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T16:22:44.661 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:44.662 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[86348]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:22:44.662 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:44.662 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:44.662 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:44.662 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T16:22:44.662 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:44 smithi064 ceph-mon[65271]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:22:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:44 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T16:22:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:45.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:45 smithi059 ceph-mon[81151]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T16:22:45.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:45 smithi059 ceph-mon[81151]: Reconfiguring daemon osd.3 on smithi059 2024-04-26T16:22:45.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:45 smithi059 ceph-mon[86348]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T16:22:45.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:45 smithi059 ceph-mon[86348]: Reconfiguring daemon osd.3 on smithi059 2024-04-26T16:22:45.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:45 smithi064 ceph-mon[65271]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T16:22:45.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:45 smithi064 ceph-mon[65271]: Reconfiguring daemon osd.3 on smithi059 2024-04-26T16:22:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[81151]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[81151]: Reconfiguring rgw.foo.smithi059.dvtzbj (monmap changed)... 2024-04-26T16:22:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:22:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[81151]: Reconfiguring daemon rgw.foo.smithi059.dvtzbj on smithi059 2024-04-26T16:22:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:46.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[86348]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:46.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:46.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:46.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[86348]: Reconfiguring rgw.foo.smithi059.dvtzbj (monmap changed)... 2024-04-26T16:22:46.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:22:46.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:46.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[86348]: Reconfiguring daemon rgw.foo.smithi059.dvtzbj on smithi059 2024-04-26T16:22:46.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:46.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:46 smithi064 ceph-mon[65271]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:46 smithi064 ceph-mon[65271]: Reconfiguring rgw.foo.smithi059.dvtzbj (monmap changed)... 2024-04-26T16:22:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:22:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:46 smithi064 ceph-mon[65271]: Reconfiguring daemon rgw.foo.smithi059.dvtzbj on smithi059 2024-04-26T16:22:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:22:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[81151]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:22:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:22:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:22:48.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[86348]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:48.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:48.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:48.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:22:48.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:22:48.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:48.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:22:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:48 smithi064 ceph-mon[65271]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:48 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:22:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:22:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:48.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:48 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:22:49.299 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:22:48.982+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:49.299 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:22:49.299 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:49.299 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:49.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:49.301 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:49.301 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:49.301 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:49.301 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:49.301 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:22:49.301 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:22:49.301 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:22:49.302 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:22:49.303 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:49.304 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:49.304 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:49.304 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:49.304 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:49.304 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:49.304 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:49.304 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:49.304 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:49.349 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:22:49.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:49 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:49.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:49 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:49.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:49 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:49.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:49 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:49.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:49 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:49.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:49 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:49.598 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:49 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T16:22:49.598 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:49 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T16:22:49.598 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:49 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:50.039 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:22:50.039 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (3m) 21s ago 25m 17.3M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:22:50.039 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (2m) 44s ago 25m 80.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:22:50.039 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 21s ago 24m 62.1M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (4m) 44s ago 30m 461M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (14m) 21s ago 31m 509M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (58s) 21s ago 31m 43.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (109s) 44s ago 30m 40.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (31s) 21s ago 30m 23.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (4m) 21s ago 26m 14.6M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (4m) 44s ago 26m 17.2M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (29m) 21s ago 29m 54.7M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (29m) 21s ago 29m 54.4M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (28m) 21s ago 28m 52.6M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (28m) 21s ago 28m 52.8M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (28m) 44s ago 28m 55.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (27m) 44s ago 27m 54.5M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:22:50.040 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (27m) 44s ago 27m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:22:50.041 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (27m) 44s ago 27m 53.6M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:22:50.041 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (3m) 44s ago 26m 66.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:22:50.041 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (24m) 21s ago 24m 77.3M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:22:50.041 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (24m) 44s ago 24m 77.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:22:50.466 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:22:50.466 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:22:50.466 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:22:50.466 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:22:50.466 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:22:50.466 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:22:50.466 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:22:50.466 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:22:50.466 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:22:50.467 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:22:50.467 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:22:50.467 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:22:50.467 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:22:50.467 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:22:50.467 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-26T16:22:50.467 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 5 2024-04-26T16:22:50.467 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:22:50.467 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:50 smithi059 ceph-mon[86348]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:50 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:22:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:50.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:50.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:50.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:22:50.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:50.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:50.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:50.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:50.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:50 smithi059 ceph-mon[81151]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:22:50.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:50 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:50.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:50.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:22:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:50 smithi059 ceph-mon[86348]: Reconfiguring mon.b (monmap changed)... 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:50 smithi059 ceph-mon[86348]: Reconfiguring daemon mon.b on smithi064 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:50 smithi059 ceph-mon[86348]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:50 smithi059 ceph-mon[81151]: Reconfiguring mon.b (monmap changed)... 2024-04-26T16:22:50.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:50 smithi059 ceph-mon[81151]: Reconfiguring daemon mon.b on smithi064 2024-04-26T16:22:50.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:50 smithi059 ceph-mon[81151]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:50.626 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:50 smithi064 ceph-mon[65271]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:50 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:50.627 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:22:50.628 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:22:50.629 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:22:50.630 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:22:50.631 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:22:50.631 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:22:50.631 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:22:50.631 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:22:50.631 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:50 smithi064 ceph-mon[65271]: Reconfiguring mon.b (monmap changed)... 2024-04-26T16:22:50.631 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:50 smithi064 ceph-mon[65271]: Reconfiguring daemon mon.b on smithi064 2024-04-26T16:22:50.631 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:50 smithi064 ceph-mon[65271]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:50.820 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:22:50.820 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:22:50.820 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:22:50.820 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi059", 2024-04-26T16:22:50.820 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [ 2024-04-26T16:22:50.820 INFO:teuthology.orchestra.run.smithi059.stdout: "mon" 2024-04-26T16:22:50.820 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T16:22:50.821 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "2/2 daemons upgraded", 2024-04-26T16:22:50.821 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading mon daemons", 2024-04-26T16:22:50.821 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:22:50.821 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:22:51.524 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:51 smithi064 ceph-mon[65271]: from='client.54174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:51.524 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:51 smithi064 ceph-mon[65271]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:51.524 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:51 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/3812511031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:22:51.524 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:51.524 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:51.524 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:51 smithi064 ceph-mon[65271]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T16:22:51.525 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:22:51.525 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:22:51.525 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:51.525 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:51 smithi064 ceph-mon[65271]: Reconfiguring daemon mgr.x on smithi064 2024-04-26T16:22:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[81151]: from='client.54174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[81151]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/3812511031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:22:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[81151]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T16:22:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:22:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:22:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[81151]: Reconfiguring daemon mgr.x on smithi064 2024-04-26T16:22:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[86348]: from='client.54174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[86348]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/3812511031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:22:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[86348]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T16:22:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T16:22:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T16:22:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:51.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:51 smithi059 ceph-mon[86348]: Reconfiguring daemon mgr.x on smithi064 2024-04-26T16:22:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[81151]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[81151]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T16:22:52.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[86348]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[86348]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T16:22:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:52 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:52.867 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:52 smithi064 ceph-mon[65271]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:22:52.867 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:52 smithi064 ceph-mon[65271]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:52.867 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:52 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:52.867 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:52 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:52.867 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:52 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T16:22:52.867 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:52 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:53 smithi059 ceph-mon[81151]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T16:22:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:53 smithi059 ceph-mon[81151]: Reconfiguring daemon osd.4 on smithi064 2024-04-26T16:22:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:53 smithi059 ceph-mon[86348]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T16:22:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:53 smithi059 ceph-mon[86348]: Reconfiguring daemon osd.4 on smithi064 2024-04-26T16:22:53.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:53 smithi064 ceph-mon[65271]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T16:22:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:53 smithi064 ceph-mon[65271]: Reconfiguring daemon osd.4 on smithi064 2024-04-26T16:22:54.564 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:22:54] "GET /metrics HTTP/1.1" 200 38079 "" "Prometheus/2.43.0" 2024-04-26T16:22:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:54 smithi064 ceph-mon[65271]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:22:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:54 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:54 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:54 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:54 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T16:22:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:54 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[86348]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:22:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T16:22:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:54.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[81151]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:22:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:22:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T16:22:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:54 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[81151]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T16:22:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[81151]: Reconfiguring daemon osd.5 on smithi064 2024-04-26T16:22:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T16:22:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[86348]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T16:22:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[86348]: Reconfiguring daemon osd.5 on smithi064 2024-04-26T16:22:55.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:55.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:55.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T16:22:55.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:55 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:55.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:55 smithi064 ceph-mon[65271]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T16:22:55.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:55 smithi064 ceph-mon[65271]: Reconfiguring daemon osd.5 on smithi064 2024-04-26T16:22:55.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:55 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:55.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:55 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:55.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:55 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T16:22:55.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:55 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:56.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:56 smithi059 ceph-mon[81151]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:56.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:56 smithi059 ceph-mon[81151]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T16:22:56.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:56 smithi059 ceph-mon[81151]: Reconfiguring daemon osd.6 on smithi064 2024-04-26T16:22:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:56 smithi059 ceph-mon[86348]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:56 smithi059 ceph-mon[86348]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T16:22:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:56 smithi059 ceph-mon[86348]: Reconfiguring daemon osd.6 on smithi064 2024-04-26T16:22:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:56 smithi064 ceph-mon[65271]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:56 smithi064 ceph-mon[65271]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T16:22:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:56 smithi064 ceph-mon[65271]: Reconfiguring daemon osd.6 on smithi064 2024-04-26T16:22:58.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[86348]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:58.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:58.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:58.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[86348]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T16:22:58.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T16:22:58.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:58.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[86348]: Reconfiguring daemon osd.7 on smithi064 2024-04-26T16:22:58.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[81151]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:58.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:58.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[81151]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T16:22:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T16:22:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:58 smithi059 ceph-mon[81151]: Reconfiguring daemon osd.7 on smithi064 2024-04-26T16:22:58.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:58 smithi064 ceph-mon[65271]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:22:58.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:58 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:58.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:58 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:58.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:58 smithi064 ceph-mon[65271]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T16:22:58.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:58 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T16:22:58.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:58 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:58.363 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:58 smithi064 ceph-mon[65271]: Reconfiguring daemon osd.7 on smithi064 2024-04-26T16:22:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:22:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:59 smithi064 ceph-mon[65271]: Reconfiguring rgw.foo.smithi064.sqspir (monmap changed)... 2024-04-26T16:22:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:22:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:22:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:59 smithi064 ceph-mon[65271]: Reconfiguring daemon rgw.foo.smithi064.sqspir on smithi064 2024-04-26T16:22:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:22:59 smithi064 ceph-mon[65271]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:00.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:00.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:00.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[86348]: Reconfiguring rgw.foo.smithi064.sqspir (monmap changed)... 2024-04-26T16:23:00.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:23:00.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:00.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[86348]: Reconfiguring daemon rgw.foo.smithi064.sqspir on smithi064 2024-04-26T16:23:00.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[86348]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:00.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:00.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:00.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[81151]: Reconfiguring rgw.foo.smithi064.sqspir (monmap changed)... 2024-04-26T16:23:00.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:23:00.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:00.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[81151]: Reconfiguring daemon rgw.foo.smithi064.sqspir on smithi064 2024-04-26T16:23:00.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:22:59 smithi059 ceph-mon[81151]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:01.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mon 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T16:23:01.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all crash 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mds 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:23:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all nfs 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all node-exporter 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all prometheus 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all alertmanager 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all grafana 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all loki 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all promtail 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Finalizing container_image settings 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:23:01.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: Upgrade: Complete! 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:01.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:23:01.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:23:01.695 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mon 2024-04-26T16:23:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T16:23:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T16:23:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all crash 2024-04-26T16:23:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mds 2024-04-26T16:23:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all nfs 2024-04-26T16:23:01.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mon 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all crash 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mds 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:23:01.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:23:01.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all nfs 2024-04-26T16:23:01.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:23:01.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all node-exporter 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all prometheus 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all alertmanager 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all grafana 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all loki 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all promtail 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Finalizing container_image settings 2024-04-26T16:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all node-exporter 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all prometheus 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all alertmanager 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all grafana 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all loki 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all promtail 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Finalizing container_image settings 2024-04-26T16:23:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:23:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:23:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:23:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:23:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:23:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:23:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:23:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:23:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:23:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:23:01.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:23:01.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:23:01.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:23:01.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:23:01.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:23:01.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:23:01.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: Upgrade: Complete! 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:23:01.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: Upgrade: Complete! 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:01.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:01.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:23:01.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:01.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:23:01.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:02.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:02 smithi064 ceph-mon[65271]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:02.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:02 smithi064 ceph-mon[65271]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:23:02.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:02 smithi064 ceph-mon[65271]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 623 B/s rd, 0 op/s 2024-04-26T16:23:02.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:02 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:23:02.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:02 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:23:03.004 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:02 smithi059 ceph-mon[86348]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:03.005 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:02 smithi059 ceph-mon[86348]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:23:03.005 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:02 smithi059 ceph-mon[86348]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 623 B/s rd, 0 op/s 2024-04-26T16:23:03.005 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:02 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:23:03.005 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:02 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:23:03.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:02 smithi059 ceph-mon[81151]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:03.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:02 smithi059 ceph-mon[81151]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:23:03.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:02 smithi059 ceph-mon[81151]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 623 B/s rd, 0 op/s 2024-04-26T16:23:03.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:02 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:23:03.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:02 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:23:03.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:23:03.257+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:03.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:23:03.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:03.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:03.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:03.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:03.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:03.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:23:03.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:03.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:23:03.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:03.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:23:03.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:03.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:03.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:03.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:03.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:03.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:03.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:03.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:03.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:03.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:03.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:03 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:03.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:03 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:03.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:03 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:23:03.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:03 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:03.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:03 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:23:03.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:03 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:04.026 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:04.026 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:04.026 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:23:04.026 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:04.026 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:23:04.027 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:04.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:04.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:04.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:23:04.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:04.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:23:04.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:03 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:04.294 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:23:04] "GET /metrics HTTP/1.1" 200 38089 "" "Prometheus/2.43.0" 2024-04-26T16:23:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:04 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:04.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:04 smithi064 ceph-mon[65271]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:04.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:04 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:04.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:04.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:04.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:04.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:04.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:04.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:04.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:23:04.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:04.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:04.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:04.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:23:04.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:04.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:04.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:04.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:04.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:04.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:04.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:04.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:04.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:04.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:23:04.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:04.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:04.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:04.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:04.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:04.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:04.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:04.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:04.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:04.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:04.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:04.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:23:04.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:04.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:04.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:04.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:23:04.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:04.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:04.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:04.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:04.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:04.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:04.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:04.946 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:04.946 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:04.946 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:04 smithi064 ceph-mon[65271]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-26T16:23:04.946 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:04 smithi064 ceph-mon[65271]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:04.946 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:04 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:23:04.946 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:04 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:23:05.033 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:05.033 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[86348]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:05.033 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.033 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:05.033 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.034 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.034 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.034 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.034 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.034 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.034 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.035 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.035 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.035 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:05.035 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.035 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.036 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.036 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.036 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.036 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.036 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.036 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.037 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.037 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.037 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:05.037 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:05.037 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:05.037 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:05.037 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:05.038 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:05.038 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:05.038 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:23:05.038 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:05.038 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:23:05.038 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:05.038 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:23:05.038 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:05.039 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:23:05.039 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:05.039 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:05.039 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.039 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:05.039 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.039 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.040 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.040 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.040 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.040 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.040 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.040 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.040 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.041 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:05.041 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.041 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.041 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.041 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.041 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.041 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.041 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.041 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.042 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.042 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[86348]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-26T16:23:05.042 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[86348]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:05.042 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:23:05.042 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:23:05.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:05.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[81151]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:05.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.043 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:05.043 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.043 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.043 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.044 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.044 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.044 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.044 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.044 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.044 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.045 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:05.045 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.045 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.045 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.045 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.045 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.045 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.046 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.046 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.046 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.046 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.046 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:05.046 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:05.047 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:05.047 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:05.047 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:05.047 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:05.047 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:05.047 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:05.048 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.049 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.049 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.049 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.049 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.049 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.049 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.049 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.049 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.049 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:05.050 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.050 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.050 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.050 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.050 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.051 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.051 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.051 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.051 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[81151]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-26T16:23:05.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[81151]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:05.052 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:23:05.052 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:04 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:23:05.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:23:05.173+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:23:05.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:05.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:23:05.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:05.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:05.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:05.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:05.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:05.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:05.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:05.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:05.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:05.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:23:05.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:05.838 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:23:05.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:05.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:23:05.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:05.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:23:05.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:05.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:05.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.839 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:05.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.840 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:05.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:05.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:05.845 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.846 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:05.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.847 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-mon[81151]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:05.848 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.848 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.848 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.848 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.848 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:05 smithi059 ceph-mon[86348]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:05.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:05 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:05.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:23:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:05.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:05.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:05.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:05.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:05 smithi064 ceph-mon[65271]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:08.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:08 smithi059 ceph-mon[81151]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-26T16:23:08.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:08 smithi059 ceph-mon[86348]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-26T16:23:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:08 smithi064 ceph-mon[65271]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-26T16:23:10.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:10 smithi059 ceph-mon[81151]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:10.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:10 smithi059 ceph-mon[86348]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:10.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:10 smithi064 ceph-mon[65271]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:12.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:12 smithi059 ceph-mon[86348]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 633 B/s rd, 0 op/s 2024-04-26T16:23:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:12 smithi059 ceph-mon[81151]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 633 B/s rd, 0 op/s 2024-04-26T16:23:12.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:12 smithi064 ceph-mon[65271]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 633 B/s rd, 0 op/s 2024-04-26T16:23:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:14 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:14 smithi059 ceph-mon[81151]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:23:14.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:23:14] "GET /metrics HTTP/1.1" 200 38089 "" "Prometheus/2.43.0" 2024-04-26T16:23:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:14 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:14 smithi059 ceph-mon[86348]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:23:14.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:14 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:14 smithi064 ceph-mon[65271]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:23:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:16 smithi064 ceph-mon[65271]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:16 smithi059 ceph-mon[86348]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:16.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:16.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:16 smithi059 ceph-mon[81151]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:16.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:18.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:18 smithi064 ceph-mon[65271]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:18.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:18 smithi059 ceph-mon[86348]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:18 smithi059 ceph-mon[81151]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:20 smithi064 ceph-mon[65271]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:20.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:20 smithi059 ceph-mon[86348]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:20.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:20 smithi059 ceph-mon[81151]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:21.941 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:23:22.573 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:22 smithi059 ceph-mon[86348]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:22.574 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:22 smithi059 ceph-mon[86348]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:22.574 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:22 smithi059 ceph-mon[81151]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:22.574 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:22 smithi059 ceph-mon[81151]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:22.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:22 smithi064 ceph-mon[65271]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:22.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:22 smithi064 ceph-mon[65271]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:23.931 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:23:23.932 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (3m) 55s ago 26m 17.3M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:23:23.932 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (3m) 78s ago 25m 80.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:23:23.932 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 55s ago 25m 62.1M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:23:23.932 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (5m) 78s ago 30m 461M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:23:23.932 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (15m) 55s ago 32m 509M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:23:23.932 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (92s) 55s ago 32m 43.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:23:23.933 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (2m) 78s ago 30m 40.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:23:23.933 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (65s) 55s ago 31m 23.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:23:23.933 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (4m) 55s ago 26m 14.6M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:23:23.933 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (4m) 78s ago 26m 17.2M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:23:23.933 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (30m) 55s ago 30m 54.7M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:23:23.933 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (29m) 55s ago 29m 54.4M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:23:23.933 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (29m) 55s ago 29m 52.6M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:23:23.934 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (29m) 55s ago 29m 52.8M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:23:23.934 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (28m) 78s ago 28m 55.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:23:23.934 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (28m) 78s ago 28m 54.5M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:23:23.934 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (28m) 78s ago 28m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:23:23.934 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (27m) 78s ago 27m 53.6M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:23:23.934 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (4m) 78s ago 26m 66.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:23:23.934 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (25m) 55s ago 25m 77.3M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:23:23.935 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (25m) 78s ago 25m 77.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:23:24.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:24 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:24 smithi059 ceph-mon[81151]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:24.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:23:24] "GET /metrics HTTP/1.1" 200 38020 "" "Prometheus/2.43.0" 2024-04-26T16:23:24.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:24 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:24.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:24 smithi059 ceph-mon[86348]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:24.644 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-04-26T16:23:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:24 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:24 smithi064 ceph-mon[65271]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:25.579 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:25 smithi059 ceph-mon[86348]: from='client.54204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:25.579 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:25 smithi059 ceph-mon[81151]: from='client.54204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:25.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:25 smithi064 ceph-mon[65271]: from='client.54204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:26.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:26 smithi059 ceph-mon[81151]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:26.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:26 smithi059 ceph-mon[86348]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:26.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:26 smithi064 ceph-mon[65271]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:26.774 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:23:27.352 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-04-26T16:23:27.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:27 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/2506887153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:27 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/2506887153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:27 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2506887153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:28.616 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:28 smithi059 ceph-mon[81151]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:28.617 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:28 smithi059 ceph-mon[86348]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:28 smithi064 ceph-mon[65271]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:29.449 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)" 2024-04-26T16:23:30.042 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-04-26T16:23:30.645 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:30 smithi059 ceph-mon[81151]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:30.645 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:30 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/123983358' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:30.645 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:30 smithi059 ceph-mon[86348]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:30.645 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:30 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/123983358' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:30.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:30 smithi064 ceph-mon[65271]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:30 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/123983358' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:31.622 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:31 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:31.622 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:31 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:31.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:31 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:32.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:32 smithi064 ceph-mon[65271]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:32.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:32 smithi059 ceph-mon[86348]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:32.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:32 smithi059 ceph-mon[81151]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:33.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:33 smithi059 ceph-mon[81151]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:33.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:33 smithi059 ceph-mon[86348]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:33.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:33 smithi064 ceph-mon[65271]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:34.267 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:23:34.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:23:34] "GET /metrics HTTP/1.1" 200 38086 "" "Prometheus/2.43.0" 2024-04-26T16:23:34.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:34 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:34.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:34 smithi059 ceph-mon[81151]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:34.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:34 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:34.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:34 smithi059 ceph-mon[86348]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:34 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:34 smithi064 ceph-mon[65271]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:34.905 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade status' 2024-04-26T16:23:36.672 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:36 smithi059 ceph-mon[86348]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:36.673 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:36 smithi059 ceph-mon[81151]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:36.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:36 smithi064 ceph-mon[65271]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:36.934 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:23:36.935 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": null, 2024-04-26T16:23:36.935 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": false, 2024-04-26T16:23:36.935 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "", 2024-04-26T16:23:36.935 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:23:36.935 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": null, 2024-04-26T16:23:36.935 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "", 2024-04-26T16:23:36.935 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:23:36.935 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:23:37.610 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:23:38.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:38 smithi059 ceph-mon[86348]: from='client.54228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:38.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:38 smithi059 ceph-mon[86348]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:38.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:38 smithi059 ceph-mon[81151]: from='client.54228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:38.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:38 smithi059 ceph-mon[81151]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:38 smithi064 ceph-mon[65271]: from='client.54228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:38 smithi064 ceph-mon[65271]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:39.921 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:23:39.921 INFO:teuthology.orchestra.run.smithi059.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:23:39.921 INFO:teuthology.orchestra.run.smithi059.stdout: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:23:40.653 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:40 smithi059 ceph-mon[81151]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:40.654 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:40 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/3414136774' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:23:40.654 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:40 smithi059 ceph-mon[86348]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:40.654 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:40 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/3414136774' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:23:40.655 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-04-26T16:23:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:40 smithi064 ceph-mon[65271]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:40 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/3414136774' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:23:42.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:42 smithi059 ceph-mon[81151]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:42.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:42 smithi059 ceph-mon[86348]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:42 smithi064 ceph-mon[65271]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:44.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:23:44] "GET /metrics HTTP/1.1" 200 38086 "" "Prometheus/2.43.0" 2024-04-26T16:23:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:44 smithi064 ceph-mon[65271]: from='client.54240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:44 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:44 smithi064 ceph-mon[65271]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:44.830 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:44 smithi059 ceph-mon[81151]: from='client.54240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:44.830 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:44 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:44.830 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:44 smithi059 ceph-mon[81151]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:44.830 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:44 smithi059 ceph-mon[86348]: from='client.54240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:44.830 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:44 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:44.830 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:44 smithi059 ceph-mon[86348]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:44.871 INFO:teuthology.orchestra.run.smithi059.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:23:45.533 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-26T16:23:46.129 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[81151]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:23:46.129 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:46.129 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:46.129 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[81151]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:46.129 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:46.129 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:23:46.129 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:46.129 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:23:46.129 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:46.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[86348]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:23:46.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:46.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:46.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[86348]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:46.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:46.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:23:46.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:46.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:23:46.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:45 smithi064 ceph-mon[65271]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:23:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:45 smithi064 ceph-mon[65271]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:23:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:23:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:23:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:47.171 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:46 smithi059 ceph-mon[81151]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 981 B/s rd, 0 op/s 2024-04-26T16:23:47.171 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:46 smithi059 ceph-mon[81151]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-04-26T16:23:47.171 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:46 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:23:47.171 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:46 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:23:47.171 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:47.171 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:46 smithi059 ceph-mon[86348]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 981 B/s rd, 0 op/s 2024-04-26T16:23:47.171 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:46 smithi059 ceph-mon[86348]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-04-26T16:23:47.171 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:46 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:23:47.171 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:46 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:23:47.171 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:47.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:46 smithi064 ceph-mon[65271]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 981 B/s rd, 0 op/s 2024-04-26T16:23:47.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:46 smithi064 ceph-mon[65271]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-04-26T16:23:47.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:46 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:23:47.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:46 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:23:47.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:23:47.695 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:23:48.398 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:23:48.398 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (4m) 80s ago 26m 17.3M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:23:48.398 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (3m) 102s ago 26m 80.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:23:48.398 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 80s ago 25m 62.1M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:23:48.398 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (5m) 102s ago 31m 461M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:23:48.398 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (15m) 80s ago 32m 509M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:23:48.398 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (116s) 80s ago 32m 43.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:23:48.398 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (2m) 102s ago 31m 40.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:23:48.398 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (89s) 80s ago 31m 23.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:23:48.398 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (5m) 80s ago 27m 14.6M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (5m) 102s ago 27m 17.2M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (30m) 80s ago 30m 54.7M 2993M 17.2.0 e1d6a67b021e e59df8cd9df9 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (30m) 80s ago 30m 54.4M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (29m) 80s ago 29m 52.6M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (29m) 80s ago 29m 52.8M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (29m) 102s ago 29m 55.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (28m) 102s ago 28m 54.5M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (28m) 102s ago 28m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (28m) 102s ago 28m 53.6M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (4m) 102s ago 27m 66.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (25m) 80s ago 25m 77.3M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:23:48.399 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (25m) 102s ago 25m 77.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:23:48.835 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:23:48.836 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-26T16:23:48.836 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 5 2024-04-26T16:23:48.836 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:23:48.836 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:23:48.882 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:23:48.766+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:48.883 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:23:48.883 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:48.883 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:48.883 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:48.883 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:48.883 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:48.883 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:23:48.883 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:48.883 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:23:48.884 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:48.884 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:23:48.884 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:48.884 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:48.884 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:48.884 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:48.884 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:48.884 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:48.884 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:48.885 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:48.886 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:48.887 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:48.888 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:48 smithi064 ceph-mon[65271]: from='client.54246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:48 smithi064 ceph-mon[65271]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:48 smithi064 ceph-mon[65271]: from='client.54252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:48 smithi064 ceph-mon[65271]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:49.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:48 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/4051834104' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:49.194 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-mon[86348]: from='client.54246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:49.194 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-mon[86348]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:49.194 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-mon[86348]: from='client.54252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:49.194 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-mon[86348]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:49.195 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/4051834104' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:49.195 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-mon[81151]: from='client.54246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:49.195 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-mon[81151]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:49.195 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-mon[81151]: from='client.54252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:49.195 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-mon[81151]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:49.195 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:48 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/4051834104' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:49.195 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:23:49.195 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", 2024-04-26T16:23:49.195 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:23:49.196 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-04-26T16:23:49.196 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:23:49.196 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "", 2024-04-26T16:23:49.196 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 image", 2024-04-26T16:23:49.196 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:23:49.196 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:23:50.126 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:49 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:50.126 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:50.126 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:50.126 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:50.126 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:50.126 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.126 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.127 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:23:50.128 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:50.129 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:50.130 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:50.130 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:50.130 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:50.130 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:50.130 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:50.130 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:50.130 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.130 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.130 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:49 smithi059 ceph-mon[81151]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:23:50.130 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:49 smithi059 ceph-mon[81151]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:49 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:50.131 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:50.132 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:23:50.133 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.134 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:49 smithi059 ceph-mon[86348]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:23:50.135 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:49 smithi059 ceph-mon[86348]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:50.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:49 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:50.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:50.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:50.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:50.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:50.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:23:50.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:23:50.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:49 smithi064 ceph-mon[65271]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:23:50.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:49 smithi064 ceph-mon[65271]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:23:51.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:50 smithi064 ceph-mon[65271]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:51.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:50 smithi059 ceph-mon[81151]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:51.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:50 smithi059 ceph-mon[86348]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:23:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:23:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mgr 2024-04-26T16:23:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mon 2024-04-26T16:23:52.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:52.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all crash 2024-04-26T16:23:52.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T16:23:52.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T16:23:52.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[81151]: Upgrade: osd.0 is safe to restart 2024-04-26T16:23:52.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mgr 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mon 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all crash 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T16:23:52.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T16:23:52.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:51 smithi059 ceph-mon[86348]: Upgrade: osd.0 is safe to restart 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mgr 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mon 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:23:52.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all crash 2024-04-26T16:23:52.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:52.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T16:23:52.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T16:23:52.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:51 smithi064 ceph-mon[65271]: Upgrade: osd.0 is safe to restart 2024-04-26T16:23:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[81151]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-04-26T16:23:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[81151]: Upgrade: Updating osd.0 2024-04-26T16:23:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T16:23:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[81151]: Deploying daemon osd.0 on smithi059 2024-04-26T16:23:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[86348]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-04-26T16:23:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[86348]: Upgrade: Updating osd.0 2024-04-26T16:23:53.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:53.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T16:23:53.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:53.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:53 smithi059 ceph-mon[86348]: Deploying daemon osd.0 on smithi059 2024-04-26T16:23:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:53 smithi064 ceph-mon[65271]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-04-26T16:23:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:53 smithi064 ceph-mon[65271]: Upgrade: Updating osd.0 2024-04-26T16:23:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:53 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:23:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:53 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T16:23:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:53 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:23:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:53 smithi064 ceph-mon[65271]: Deploying daemon osd.0 on smithi059 2024-04-26T16:23:54.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:23:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:23:54] "GET /metrics HTTP/1.1" 200 38090 "" "Prometheus/2.43.0" 2024-04-26T16:23:54.614 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:54 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:54.614 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:54 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:54.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:54 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:23:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:55 smithi059 ceph-mon[81151]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:55 smithi059 ceph-mon[86348]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:55.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:55 smithi064 ceph-mon[65271]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:23:56.107 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:55 smithi059 systemd[1]: Stopping Ceph osd.0 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:23:56.499 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[29410]: 2024-04-26T16:23:56.106+0000 7fd0c3ae3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:23:56.499 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[29410]: 2024-04-26T16:23:56.106+0000 7fd0c3ae3700 -1 osd.0 107 *** Got signal Terminated *** 2024-04-26T16:23:56.499 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[29410]: 2024-04-26T16:23:56.106+0000 7fd0c3ae3700 -1 osd.0 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:23:56.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:56 smithi059 ceph-mon[81151]: osd.0 marked itself down and dead 2024-04-26T16:23:56.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:56 smithi059 ceph-mon[86348]: osd.0 marked itself down and dead 2024-04-26T16:23:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:56 smithi064 ceph-mon[65271]: osd.0 marked itself down and dead 2024-04-26T16:23:57.315 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:57 smithi059 podman[93832]: 2024-04-26 16:23:57.041728452 +0000 UTC m=+1.032076898 container died e59df8cd9df94d2e829c78eb5bacb67b6cda42c7504c9f668d16440862e1b794 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vcs-type=git, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, distribution-scope=public, architecture=x86_64, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , release=754, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8) 2024-04-26T16:23:57.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:57 smithi059 ceph-mon[86348]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:23:57.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:57 smithi059 ceph-mon[86348]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:23:57.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:57 smithi059 ceph-mon[86348]: osdmap e108: 8 total, 7 up, 8 in 2024-04-26T16:23:57.583 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:57 smithi059 podman[93832]: 2024-04-26 16:23:57.423409507 +0000 UTC m=+1.413757944 container cleanup e59df8cd9df94d2e829c78eb5bacb67b6cda42c7504c9f668d16440862e1b794 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, version=8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, RELEASE=HEAD, maintainer=Guillaume Abrioux , ceph=True, com.redhat.component=centos-stream-container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_CLEAN=True) 2024-04-26T16:23:57.583 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:57 smithi059 bash[93832]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0 2024-04-26T16:23:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:57 smithi059 ceph-mon[81151]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:23:57.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:57 smithi059 ceph-mon[81151]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:23:57.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:57 smithi059 ceph-mon[81151]: osdmap e108: 8 total, 7 up, 8 in 2024-04-26T16:23:57.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:57 smithi064 ceph-mon[65271]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:23:57.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:57 smithi064 ceph-mon[65271]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:23:57.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:57 smithi064 ceph-mon[65271]: osdmap e108: 8 total, 7 up, 8 in 2024-04-26T16:23:58.006 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:57 smithi059 podman[93860]: 2024-04-26 16:23:57.683451984 +0000 UTC m=+0.638820801 container remove e59df8cd9df94d2e829c78eb5bacb67b6cda42c7504c9f668d16440862e1b794 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, ceph=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_CLEAN=True, vendor=Red Hat, Inc., io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, name=centos-stream, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_BRANCH=HEAD, RELEASE=HEAD) 2024-04-26T16:23:58.260 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:58 smithi059 podman[93953]: 2024-04-26 16:23:57.910480097 +0000 UTC m=+0.018539385 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:23:58.261 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:58 smithi059 podman[93953]: 2024-04-26 16:23:58.076186613 +0000 UTC m=+0.184245885 container create 18c4ec7c63fdf1b65d92cb1a2da79ed7ece5b0265e4f731277a13c8ea783b061 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0) 2024-04-26T16:23:58.511 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:58 smithi059 podman[93953]: 2024-04-26 16:23:58.261680185 +0000 UTC m=+0.369739458 container init 18c4ec7c63fdf1b65d92cb1a2da79ed7ece5b0265e4f731277a13c8ea783b061 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T16:23:58.512 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:58 smithi059 podman[93953]: 2024-04-26 16:23:58.265089285 +0000 UTC m=+0.373148558 container start 18c4ec7c63fdf1b65d92cb1a2da79ed7ece5b0265e4f731277a13c8ea783b061 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:23:58.512 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:58 smithi059 podman[93953]: 2024-04-26 16:23:58.353444009 +0000 UTC m=+0.461503282 container attach 18c4ec7c63fdf1b65d92cb1a2da79ed7ece5b0265e4f731277a13c8ea783b061 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-26T16:23:58.832 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:58 smithi059 podman[93953]: 2024-04-26 16:23:58.576768769 +0000 UTC m=+0.684828051 container died 18c4ec7c63fdf1b65d92cb1a2da79ed7ece5b0265e4f731277a13c8ea783b061 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux ) 2024-04-26T16:23:58.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:58 smithi059 ceph-mon[81151]: osdmap e109: 8 total, 7 up, 8 in 2024-04-26T16:23:58.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:58 smithi059 ceph-mon[86348]: osdmap e109: 8 total, 7 up, 8 in 2024-04-26T16:23:58.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:58 smithi064 ceph-mon[65271]: osdmap e109: 8 total, 7 up, 8 in 2024-04-26T16:23:59.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:59 smithi059 ceph-mon[86348]: pgmap v71: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:23:59.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:23:59 smithi059 ceph-mon[86348]: Health check failed: Reduced data availability: 11 pgs inactive, 52 pgs peering (PG_AVAILABILITY) 2024-04-26T16:23:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:59 smithi059 ceph-mon[81151]: pgmap v71: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:23:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:23:59 smithi059 ceph-mon[81151]: Health check failed: Reduced data availability: 11 pgs inactive, 52 pgs peering (PG_AVAILABILITY) 2024-04-26T16:23:59.833 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:59 smithi059 podman[93978]: 2024-04-26 16:23:59.381061239 +0000 UTC m=+0.798844152 container remove 18c4ec7c63fdf1b65d92cb1a2da79ed7ece5b0265e4f731277a13c8ea783b061 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=) 2024-04-26T16:23:59.834 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:59 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-26T16:23:59.834 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:59 smithi059 systemd[1]: Stopped Ceph osd.0 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:23:59.834 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:59 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.0.service: Consumed 15.387s CPU time. 2024-04-26T16:23:59.834 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:59 smithi059 systemd[1]: Starting Ceph osd.0 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:23:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:59 smithi064 ceph-mon[65271]: pgmap v71: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:23:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:23:59 smithi064 ceph-mon[65271]: Health check failed: Reduced data availability: 11 pgs inactive, 52 pgs peering (PG_AVAILABILITY) 2024-04-26T16:24:00.086 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:59 smithi059 podman[94099]: 2024-04-26 16:23:59.738771382 +0000 UTC m=+0.018783040 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:24:00.086 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:23:59 smithi059 podman[94099]: 2024-04-26 16:23:59.85937845 +0000 UTC m=+0.139390143 container create 84c119207adc9bb700714f36e3146aef35e0568eeadc2c83cf642b6a7654069b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:24:00.583 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 podman[94099]: 2024-04-26 16:24:00.087077768 +0000 UTC m=+0.367089428 container init 84c119207adc9bb700714f36e3146aef35e0568eeadc2c83cf642b6a7654069b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T16:24:00.583 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 podman[94099]: 2024-04-26 16:24:00.090728397 +0000 UTC m=+0.370740050 container start 84c119207adc9bb700714f36e3146aef35e0568eeadc2c83cf642b6a7654069b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:24:00.583 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 podman[94099]: 2024-04-26 16:24:00.169966306 +0000 UTC m=+0.449977966 container attach 84c119207adc9bb700714f36e3146aef35e0568eeadc2c83cf642b6a7654069b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-26T16:24:00.906 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate[94113]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:24:00.906 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate[94113]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:24:00.906 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 bash[94099]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:24:00.906 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 bash[94099]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:24:00.906 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 bash[94099]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 bash[94099]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 bash[94099]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 bash[94099]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 bash[94099]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 bash[94099]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 bash[94099]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate[94113]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate[94113]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate[94113]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate[94113]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate[94113]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate[94113]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate[94113]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T16:24:00.907 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:00 smithi059 podman[94099]: 2024-04-26 16:24:00.611917113 +0000 UTC m=+0.891928761 container died 84c119207adc9bb700714f36e3146aef35e0568eeadc2c83cf642b6a7654069b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True) 2024-04-26T16:24:01.515 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:01 smithi059 podman[94254]: 2024-04-26 16:24:01.215895821 +0000 UTC m=+0.598515996 container remove 84c119207adc9bb700714f36e3146aef35e0568eeadc2c83cf642b6a7654069b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424) 2024-04-26T16:24:01.516 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:01 smithi059 podman[94315]: 2024-04-26 16:24:01.335620484 +0000 UTC m=+0.018556561 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:24:01.516 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:01 smithi059 podman[94315]: 2024-04-26 16:24:01.450305972 +0000 UTC m=+0.133242083 container create 28dcbd084295d4b3767a1988e13add78aeb11bd50f129ce147cf3f0e2a01311b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:24:01.768 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:01 smithi059 ceph-mon[81151]: pgmap v72: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:24:01.769 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:01.769 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:01.769 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:01 smithi059 ceph-mon[86348]: pgmap v72: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:24:01.769 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:01.769 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:01.769 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:01 smithi059 podman[94315]: 2024-04-26 16:24:01.669302334 +0000 UTC m=+0.352238399 container init 28dcbd084295d4b3767a1988e13add78aeb11bd50f129ce147cf3f0e2a01311b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-26T16:24:01.769 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:01 smithi059 podman[94315]: 2024-04-26 16:24:01.672372434 +0000 UTC m=+0.355308501 container start 28dcbd084295d4b3767a1988e13add78aeb11bd50f129ce147cf3f0e2a01311b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:24:01.770 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:01 smithi059 ceph-osd[94334]: -- 172.21.15.59:0/847792851 <== mon.1 v2:172.21.15.59:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f6eead9680 con 0x55f6eeaa3400 2024-04-26T16:24:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:01 smithi064 ceph-mon[65271]: pgmap v72: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-26T16:24:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:02.082 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:01 smithi059 bash[94315]: 28dcbd084295d4b3767a1988e13add78aeb11bd50f129ce147cf3f0e2a01311b 2024-04-26T16:24:02.082 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:01 smithi059 systemd[1]: Started Ceph osd.0 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:24:02.583 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[94330]: 2024-04-26T16:24:02.211+0000 7fd1fe5f57c0 -1 Falling back to public interface 2024-04-26T16:24:03.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-mon[81151]: pgmap v73: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-26T16:24:03.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:03.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:03.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:03.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-mon[86348]: pgmap v73: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-26T16:24:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:03.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:02 smithi064 ceph-mon[65271]: pgmap v73: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-26T16:24:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:04.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:03 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:04.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:03 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:04.088 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:03 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:04.445 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:24:04] "GET /metrics HTTP/1.1" 200 38220 "" "Prometheus/2.43.0" 2024-04-26T16:24:05.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:04 smithi059 ceph-mon[86348]: pgmap v74: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-04-26T16:24:05.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:04 smithi059 ceph-mon[86348]: Health check failed: Degraded data redundancy: 69/627 objects degraded (11.005%), 21 pgs degraded (PG_DEGRADED) 2024-04-26T16:24:05.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:04 smithi059 ceph-mon[86348]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 52 pgs peering) 2024-04-26T16:24:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:04 smithi059 ceph-mon[81151]: pgmap v74: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-04-26T16:24:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:04 smithi059 ceph-mon[81151]: Health check failed: Degraded data redundancy: 69/627 objects degraded (11.005%), 21 pgs degraded (PG_DEGRADED) 2024-04-26T16:24:05.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:04 smithi059 ceph-mon[81151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 52 pgs peering) 2024-04-26T16:24:05.083 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[94330]: 2024-04-26T16:24:04.930+0000 7fd1fe5f57c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-26T16:24:05.083 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[94330]: 2024-04-26T16:24:04.969+0000 7fd1fe5f57c0 -1 osd.0 107 log_to_monitors true 2024-04-26T16:24:05.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:04 smithi064 ceph-mon[65271]: pgmap v74: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-04-26T16:24:05.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:04 smithi064 ceph-mon[65271]: Health check failed: Degraded data redundancy: 69/627 objects degraded (11.005%), 21 pgs degraded (PG_DEGRADED) 2024-04-26T16:24:05.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:04 smithi064 ceph-mon[65271]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 52 pgs peering) 2024-04-26T16:24:06.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:05 smithi059 ceph-mon[86348]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T16:24:06.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:05 smithi059 ceph-mon[81151]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T16:24:06.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:05 smithi064 ceph-mon[65271]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T16:24:06.582 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[94330]: 2024-04-26T16:24:06.310+0000 7fd1eba45700 -1 osd.0 107 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T16:24:07.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:06 smithi064 ceph-mon[65271]: pgmap v75: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-04-26T16:24:07.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:06 smithi064 ceph-mon[65271]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T16:24:07.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:06 smithi064 ceph-mon[65271]: osdmap e110: 8 total, 7 up, 8 in 2024-04-26T16:24:07.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:06 smithi064 ceph-mon[65271]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:24:07.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:06 smithi064 ceph-mon[65271]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' 2024-04-26T16:24:07.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:06 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:07.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:06 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:07.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[81151]: pgmap v75: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-04-26T16:24:07.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[81151]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T16:24:07.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[81151]: osdmap e110: 8 total, 7 up, 8 in 2024-04-26T16:24:07.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[81151]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:24:07.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[81151]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' 2024-04-26T16:24:07.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:07.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:07.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[86348]: pgmap v75: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-04-26T16:24:07.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[86348]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T16:24:07.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[86348]: osdmap e110: 8 total, 7 up, 8 in 2024-04-26T16:24:07.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[86348]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:24:07.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[86348]: from='osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245]' entity='osd.0' 2024-04-26T16:24:07.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:07.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:06 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:08 smithi064 ceph-mon[65271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:24:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:08 smithi064 ceph-mon[65271]: osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245] boot 2024-04-26T16:24:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:08 smithi064 ceph-mon[65271]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T16:24:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:08 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:24:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:08 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:08 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[86348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[86348]: osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245] boot 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[86348]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[81151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[81151]: osd.0 [v2:172.21.15.59:6802/127823245,v1:172.21.15.59:6803/127823245] boot 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[81151]: osdmap e111: 8 total, 8 up, 8 in 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T16:24:08.441 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:08.442 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:08 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:09 smithi059 ceph-mon[81151]: pgmap v78: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-04-26T16:24:09.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:09 smithi059 ceph-mon[81151]: osdmap e112: 8 total, 8 up, 8 in 2024-04-26T16:24:09.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:09 smithi059 ceph-mon[86348]: pgmap v78: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-04-26T16:24:09.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:09 smithi059 ceph-mon[86348]: osdmap e112: 8 total, 8 up, 8 in 2024-04-26T16:24:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:09 smithi064 ceph-mon[65271]: pgmap v78: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-04-26T16:24:09.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:09 smithi064 ceph-mon[65271]: osdmap e112: 8 total, 8 up, 8 in 2024-04-26T16:24:11.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[86348]: pgmap v80: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 69/627 objects degraded (11.005%) 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[81151]: pgmap v80: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 69/627 objects degraded (11.005%) 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:11.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:11.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:11.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:11.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:11 smithi064 ceph-mon[65271]: pgmap v80: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 69/627 objects degraded (11.005%) 2024-04-26T16:24:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-mon[86348]: pgmap v81: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 0 op/s; 69/627 objects degraded (11.005%) 2024-04-26T16:24:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-mon[86348]: pgmap v82: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s; 69/627 objects degraded (11.005%) 2024-04-26T16:24:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-mon[81151]: pgmap v81: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 0 op/s; 69/627 objects degraded (11.005%) 2024-04-26T16:24:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-mon[81151]: pgmap v82: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s; 69/627 objects degraded (11.005%) 2024-04-26T16:24:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:12 smithi064 ceph-mon[65271]: pgmap v81: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 0 op/s; 69/627 objects degraded (11.005%) 2024-04-26T16:24:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:12 smithi064 ceph-mon[65271]: pgmap v82: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s; 69/627 objects degraded (11.005%) 2024-04-26T16:24:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:12 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:12 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:13.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:24:12.754+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:13.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:24:13.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:13.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:13.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:13.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:13.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:13.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:24:13.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:13.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:13.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:13.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:12 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:24:12.766+0000 7fe2d86fd700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-04-26T16:24:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:13 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:13 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:13 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:13 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:13 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:13.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:13 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:13.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:13 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:13.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:13 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:13.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:13 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:13.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:13 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:13 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:13 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:13 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:13 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:13 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:14.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:24:14] "GET /metrics HTTP/1.1" 200 38220 "" "Prometheus/2.43.0" 2024-04-26T16:24:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:14.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:14.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:14.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:14.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:24:14.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[86348]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[86348]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 915 B/s rd, 0 op/s 2024-04-26T16:24:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[86348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/627 objects degraded (11.005%), 21 pgs degraded) 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[81151]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[81151]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 915 B/s rd, 0 op/s 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:14 smithi059 ceph-mon[81151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/627 objects degraded (11.005%), 21 pgs degraded) 2024-04-26T16:24:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:14 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:14.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:24:14.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:14.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:14.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:14.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:14.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:14.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:14.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:14.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:14.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:14.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:14.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:24:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:14.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:14.693 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:14.694 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:14.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:14 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:14.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:14 smithi064 ceph-mon[65271]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-04-26T16:24:14.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:14 smithi064 ceph-mon[65271]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 915 B/s rd, 0 op/s 2024-04-26T16:24:14.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:14 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:14.694 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:14 smithi064 ceph-mon[65271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/627 objects degraded (11.005%), 21 pgs degraded) 2024-04-26T16:24:16.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:16 smithi059 ceph-mon[86348]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 712 B/s rd, 0 op/s 2024-04-26T16:24:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:16 smithi059 ceph-mon[81151]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 712 B/s rd, 0 op/s 2024-04-26T16:24:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:16 smithi064 ceph-mon[65271]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 712 B/s rd, 0 op/s 2024-04-26T16:24:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:18.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:18 smithi059 ceph-mon[81151]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 0 op/s 2024-04-26T16:24:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:18 smithi059 ceph-mon[86348]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 0 op/s 2024-04-26T16:24:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:18 smithi064 ceph-mon[65271]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 0 op/s 2024-04-26T16:24:19.580 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:24:20.272 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:24:20.272 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (4m) 13s ago 27m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:24:20.272 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (4m) 2m ago 26m 80.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:24:20.272 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 13s ago 26m 62.1M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:24:20.272 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (6m) 2m ago 31m 461M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:24:20.272 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (16m) 13s ago 33m 518M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:24:20.272 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (2m) 13s ago 33m 49.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:24:20.272 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (3m) 2m ago 31m 40.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (2m) 13s ago 31m 40.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (5m) 13s ago 27m 16.7M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (5m) 2m ago 27m 17.2M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (18s) 13s ago 31m 12.0M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (30m) 13s ago 30m 55.4M 2993M 17.2.0 e1d6a67b021e 0f648ce3260c 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (30m) 13s ago 30m 53.2M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (30m) 13s ago 30m 53.5M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (29m) 2m ago 29m 55.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (29m) 2m ago 29m 54.5M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (28m) 2m ago 28m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (28m) 2m ago 28m 53.6M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:24:20.273 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (5m) 2m ago 27m 66.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:24:20.274 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (26m) 13s ago 26m 77.6M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:24:20.274 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (26m) 2m ago 26m 77.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:24:20.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:20 smithi059 ceph-mon[86348]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:24:20.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:20 smithi059 ceph-mon[86348]: from='client.54282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:20 smithi059 ceph-mon[81151]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:24:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:20 smithi059 ceph-mon[81151]: from='client.54282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:20 smithi064 ceph-mon[65271]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:24:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:20 smithi064 ceph-mon[65271]: from='client.54282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:20.700 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:24:20.700 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:24:20.700 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:24:20.700 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:24:20.700 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:24:20.700 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:24:20.700 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:24:20.700 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:24:20.700 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-26T16:24:20.700 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 1 2024-04-26T16:24:20.701 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:24:20.701 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:24:20.701 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:24:20.701 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:24:20.701 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:24:20.701 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-26T16:24:20.701 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 6 2024-04-26T16:24:20.701 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:24:20.701 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:24:21.052 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:24:21.053 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:24:21.053 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:24:21.053 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-04-26T16:24:21.053 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:24:21.053 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "1/8 daemons upgraded", 2024-04-26T16:24:21.053 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T16:24:21.053 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:24:21.053 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:24:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:21 smithi064 ceph-mon[65271]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:21 smithi064 ceph-mon[65271]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:21 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2694637882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:21.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:21 smithi059 ceph-mon[81151]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:21.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:21 smithi059 ceph-mon[81151]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:21.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:21 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/2694637882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:21.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:21.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:21 smithi059 ceph-mon[86348]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:21.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:21 smithi059 ceph-mon[86348]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:21.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:21 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/2694637882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:21.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:22.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:22 smithi064 ceph-mon[65271]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:24:22.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:22 smithi064 ceph-mon[65271]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:22.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:22 smithi059 ceph-mon[81151]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:24:22.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:22 smithi059 ceph-mon[81151]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:22.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:22 smithi059 ceph-mon[86348]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:24:22.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:22 smithi059 ceph-mon[86348]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:24 smithi059 ceph-mon[81151]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:24 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:24.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:24:24] "GET /metrics HTTP/1.1" 200 38243 "" "Prometheus/2.43.0" 2024-04-26T16:24:24.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:24 smithi059 ceph-mon[86348]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:24.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:24 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:24.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:24 smithi064 ceph-mon[65271]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:24 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:26.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:26 smithi064 ceph-mon[65271]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:24:26.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:26 smithi059 ceph-mon[81151]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:24:26.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:26 smithi059 ceph-mon[86348]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:24:28.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:28 smithi059 ceph-mon[81151]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:24:28.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:28 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:28.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:28 smithi059 ceph-mon[86348]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:24:28.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:28 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:28.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:28 smithi064 ceph-mon[65271]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:24:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:28 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:29 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:29 smithi064 ceph-mon[65271]: Upgrade: osd.1 is safe to restart 2024-04-26T16:24:29.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:29 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:29.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:29 smithi059 ceph-mon[81151]: Upgrade: osd.1 is safe to restart 2024-04-26T16:24:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:29 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T16:24:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:29 smithi059 ceph-mon[86348]: Upgrade: osd.1 is safe to restart 2024-04-26T16:24:30.708 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[81151]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:30.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[81151]: Upgrade: Updating osd.1 2024-04-26T16:24:30.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:30.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T16:24:30.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:30.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[81151]: Deploying daemon osd.1 on smithi059 2024-04-26T16:24:30.709 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[86348]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:30.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[86348]: Upgrade: Updating osd.1 2024-04-26T16:24:30.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:30.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T16:24:30.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:30.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:30 smithi059 ceph-mon[86348]: Deploying daemon osd.1 on smithi059 2024-04-26T16:24:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:30 smithi064 ceph-mon[65271]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:30 smithi064 ceph-mon[65271]: Upgrade: Updating osd.1 2024-04-26T16:24:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T16:24:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:30 smithi064 ceph-mon[65271]: Deploying daemon osd.1 on smithi059 2024-04-26T16:24:31.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:31 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:31.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:31 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:31 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:32.722 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:32 smithi059 ceph-mon[81151]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:24:32.723 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:32 smithi059 ceph-mon[86348]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:24:32.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:32 smithi064 ceph-mon[65271]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:24:32.976 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:32 smithi059 systemd[1]: Stopping Ceph osd.1 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:24:33.332 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[32369]: 2024-04-26T16:24:33.021+0000 7f35ca1d1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:24:33.333 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[32369]: 2024-04-26T16:24:33.021+0000 7f35ca1d1700 -1 osd.1 112 *** Got signal Terminated *** 2024-04-26T16:24:33.333 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[32369]: 2024-04-26T16:24:33.021+0000 7f35ca1d1700 -1 osd.1 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:24:33.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:33 smithi059 ceph-mon[81151]: osd.1 marked itself down and dead 2024-04-26T16:24:33.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:33 smithi059 ceph-mon[86348]: osd.1 marked itself down and dead 2024-04-26T16:24:33.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:33 smithi064 ceph-mon[65271]: osd.1 marked itself down and dead 2024-04-26T16:24:34.138 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:33 smithi059 podman[98766]: 2024-04-26 16:24:33.836750876 +0000 UTC m=+0.950132650 container died 0f648ce3260c2acf0fec601f946a0b2d3c6a239a9439fc6a1d8345796433f9f3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, vcs-type=git, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.k8s.display-name=CentOS Stream 8, release=754, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.buildah.version=1.19.8) 2024-04-26T16:24:34.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:24:34] "GET /metrics HTTP/1.1" 200 38243 "" "Prometheus/2.43.0" 2024-04-26T16:24:34.484 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:34 smithi059 podman[98766]: 2024-04-26 16:24:34.338690781 +0000 UTC m=+1.452072545 container cleanup 0f648ce3260c2acf0fec601f946a0b2d3c6a239a9439fc6a1d8345796433f9f3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.openshift.expose-services=) 2024-04-26T16:24:34.484 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:34 smithi059 bash[98766]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1 2024-04-26T16:24:34.832 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:34 smithi059 podman[98811]: 2024-04-26 16:24:34.582282306 +0000 UTC m=+0.745852859 container remove 0f648ce3260c2acf0fec601f946a0b2d3c6a239a9439fc6a1d8345796433f9f3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vcs-type=git, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, maintainer=Guillaume Abrioux , RELEASE=HEAD, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-26T16:24:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:34 smithi059 ceph-mon[81151]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:34 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:34 smithi059 ceph-mon[81151]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:24:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:34 smithi059 ceph-mon[81151]: osdmap e113: 8 total, 7 up, 8 in 2024-04-26T16:24:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:34 smithi059 ceph-mon[86348]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:34 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:34 smithi059 ceph-mon[86348]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:24:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:34 smithi059 ceph-mon[86348]: osdmap e113: 8 total, 7 up, 8 in 2024-04-26T16:24:34.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:34 smithi064 ceph-mon[65271]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:34.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:34 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:34.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:34 smithi064 ceph-mon[65271]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:24:34.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:34 smithi064 ceph-mon[65271]: osdmap e113: 8 total, 7 up, 8 in 2024-04-26T16:24:35.332 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:35 smithi059 podman[98907]: 2024-04-26 16:24:34.925815851 +0000 UTC m=+0.019337385 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:24:35.333 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:35 smithi059 podman[98907]: 2024-04-26 16:24:35.049749349 +0000 UTC m=+0.143270864 container create a9074b090a2cad37b7b9914101f79727dc091f29efcb1acae2a35eb3d9f23cae (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T16:24:35.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:35 smithi059 ceph-mon[86348]: osdmap e114: 8 total, 7 up, 8 in 2024-04-26T16:24:35.795 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:35 smithi059 ceph-mon[81151]: osdmap e114: 8 total, 7 up, 8 in 2024-04-26T16:24:35.795 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:35 smithi059 podman[98907]: 2024-04-26 16:24:35.361247143 +0000 UTC m=+0.454768675 container init a9074b090a2cad37b7b9914101f79727dc091f29efcb1acae2a35eb3d9f23cae (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:24:35.795 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:35 smithi059 podman[98907]: 2024-04-26 16:24:35.365073379 +0000 UTC m=+0.458594899 container start a9074b090a2cad37b7b9914101f79727dc091f29efcb1acae2a35eb3d9f23cae (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T16:24:35.795 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:35 smithi059 podman[98907]: 2024-04-26 16:24:35.452404227 +0000 UTC m=+0.545925744 container attach a9074b090a2cad37b7b9914101f79727dc091f29efcb1acae2a35eb3d9f23cae (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5) 2024-04-26T16:24:35.795 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:35 smithi059 podman[98907]: 2024-04-26 16:24:35.674203455 +0000 UTC m=+0.767724977 container died a9074b090a2cad37b7b9914101f79727dc091f29efcb1acae2a35eb3d9f23cae (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:24:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:35 smithi064 ceph-mon[65271]: osdmap e114: 8 total, 7 up, 8 in 2024-04-26T16:24:36.581 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:36 smithi059 ceph-mon[86348]: pgmap v96: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:36.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:36 smithi059 ceph-mon[81151]: pgmap v96: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:36.582 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:36 smithi059 podman[98932]: 2024-04-26 16:24:36.331611366 +0000 UTC m=+0.651542215 container remove a9074b090a2cad37b7b9914101f79727dc091f29efcb1acae2a35eb3d9f23cae (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5) 2024-04-26T16:24:36.582 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:36 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-26T16:24:36.582 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:36 smithi059 systemd[1]: Stopped Ceph osd.1 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:24:36.582 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:36 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.1.service: Consumed 16.763s CPU time. 2024-04-26T16:24:36.582 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:36 smithi059 systemd[1]: Starting Ceph osd.1 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:24:36.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:36 smithi064 ceph-mon[65271]: pgmap v96: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:24:37.083 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:36 smithi059 podman[99037]: 2024-04-26 16:24:36.703119473 +0000 UTC m=+0.018864881 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:24:37.083 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:36 smithi059 podman[99037]: 2024-04-26 16:24:36.816054706 +0000 UTC m=+0.131800160 container create 6b3d25423bf4ab692bf6251b5fcdc732dfd9b9648f11ec16aae0dab1210b581c (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:24:37.083 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 podman[99037]: 2024-04-26 16:24:37.044217298 +0000 UTC m=+0.359962712 container init 6b3d25423bf4ab692bf6251b5fcdc732dfd9b9648f11ec16aae0dab1210b581c (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T16:24:37.083 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 podman[99037]: 2024-04-26 16:24:37.047888065 +0000 UTC m=+0.363633484 container start 6b3d25423bf4ab692bf6251b5fcdc732dfd9b9648f11ec16aae0dab1210b581c (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-26T16:24:37.542 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 podman[99037]: 2024-04-26 16:24:37.143590183 +0000 UTC m=+0.459335592 container attach 6b3d25423bf4ab692bf6251b5fcdc732dfd9b9648f11ec16aae0dab1210b581c (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:24:37.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-mon[81151]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-04-26T16:24:37.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-mon[86348]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-04-26T16:24:37.833 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate[99052]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:24:37.833 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate[99052]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:24:37.833 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 bash[99037]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:24:37.833 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 bash[99037]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 bash[99037]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 bash[99037]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 bash[99037]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 bash[99037]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 bash[99037]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 bash[99037]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 bash[99037]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate[99052]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate[99052]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate[99052]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate[99052]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate[99052]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T16:24:37.834 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate[99052]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T16:24:37.835 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate[99052]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T16:24:37.835 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:37 smithi059 podman[99037]: 2024-04-26 16:24:37.570450193 +0000 UTC m=+0.886195600 container died 6b3d25423bf4ab692bf6251b5fcdc732dfd9b9648f11ec16aae0dab1210b581c (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:24:37.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:37 smithi064 ceph-mon[65271]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-04-26T16:24:38.523 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:38 smithi059 podman[99176]: 2024-04-26 16:24:38.189469172 +0000 UTC m=+0.613503449 container remove 6b3d25423bf4ab692bf6251b5fcdc732dfd9b9648f11ec16aae0dab1210b581c (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-26T16:24:38.523 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:38 smithi059 podman[99236]: 2024-04-26 16:24:38.313356452 +0000 UTC m=+0.019132998 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:24:38.523 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:38 smithi059 podman[99236]: 2024-04-26 16:24:38.523773287 +0000 UTC m=+0.229549804 container create 7e91e27615212cb6e3c8d1c9abc93828c233f8b56b9d45c02b9211aac09cf13e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:24:38.783 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:38 smithi059 ceph-mon[86348]: pgmap v97: 161 pgs: 1 active+undersized+degraded, 20 stale+active+clean, 6 active+undersized, 134 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 6/627 objects degraded (0.957%) 2024-04-26T16:24:38.783 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:38 smithi059 ceph-mon[81151]: pgmap v97: 161 pgs: 1 active+undersized+degraded, 20 stale+active+clean, 6 active+undersized, 134 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 6/627 objects degraded (0.957%) 2024-04-26T16:24:38.784 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:38 smithi059 podman[99236]: 2024-04-26 16:24:38.759369989 +0000 UTC m=+0.465146512 container init 7e91e27615212cb6e3c8d1c9abc93828c233f8b56b9d45c02b9211aac09cf13e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:24:38.784 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:38 smithi059 podman[99236]: 2024-04-26 16:24:38.762558935 +0000 UTC m=+0.468335457 container start 7e91e27615212cb6e3c8d1c9abc93828c233f8b56b9d45c02b9211aac09cf13e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:24:38.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:38 smithi064 ceph-mon[65271]: pgmap v97: 161 pgs: 1 active+undersized+degraded, 20 stale+active+clean, 6 active+undersized, 134 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 6/627 objects degraded (0.957%) 2024-04-26T16:24:39.082 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:38 smithi059 ceph-osd[99254]: -- 172.21.15.59:0/3170153606 <== mon.1 v2:172.21.15.59:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d0f5a65680 con 0x55d0f5a2ec00 2024-04-26T16:24:39.083 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:38 smithi059 bash[99236]: 7e91e27615212cb6e3c8d1c9abc93828c233f8b56b9d45c02b9211aac09cf13e 2024-04-26T16:24:39.083 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:38 smithi059 systemd[1]: Started Ceph osd.1 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:24:39.583 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[99250]: 2024-04-26T16:24:39.307+0000 7f158f9137c0 -1 Falling back to public interface 2024-04-26T16:24:40.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-mon[81151]: pgmap v98: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:40.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:40.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:40.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:40.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-mon[86348]: pgmap v98: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:40.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:39 smithi064 ceph-mon[65271]: pgmap v98: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:42.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:41 smithi064 ceph-mon[65271]: pgmap v99: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:42.288 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:41 smithi059 ceph-mon[86348]: pgmap v99: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:42.288 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[99250]: 2024-04-26T16:24:42.047+0000 7f158f9137c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-26T16:24:42.288 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[99250]: 2024-04-26T16:24:42.074+0000 7f158f9137c0 -1 osd.1 112 log_to_monitors true 2024-04-26T16:24:42.288 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:41 smithi059 ceph-mon[81151]: pgmap v99: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:43.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:42 smithi059 ceph-mon[81151]: from='osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T16:24:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:42 smithi059 ceph-mon[86348]: from='osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T16:24:43.083 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:24:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[99250]: 2024-04-26T16:24:42.947+0000 7f1580d6b700 -1 osd.1 112 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T16:24:43.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:42 smithi064 ceph-mon[65271]: from='osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T16:24:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:43 smithi064 ceph-mon[65271]: pgmap v100: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:43 smithi064 ceph-mon[65271]: from='osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T16:24:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:43 smithi064 ceph-mon[65271]: osdmap e115: 8 total, 7 up, 8 in 2024-04-26T16:24:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:43 smithi064 ceph-mon[65271]: from='osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:24:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:43 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:44.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:24:44] "GET /metrics HTTP/1.1" 200 38243 "" "Prometheus/2.43.0" 2024-04-26T16:24:44.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[86348]: pgmap v100: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:44.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[86348]: from='osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T16:24:44.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[86348]: osdmap e115: 8 total, 7 up, 8 in 2024-04-26T16:24:44.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[86348]: from='osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:24:44.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:44.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:44.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:44.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[81151]: pgmap v100: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:44.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[81151]: from='osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T16:24:44.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[81151]: osdmap e115: 8 total, 7 up, 8 in 2024-04-26T16:24:44.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[81151]: from='osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:24:44.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:44.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:44.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:45.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[81151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:24:45.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[81151]: osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033] boot 2024-04-26T16:24:45.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[81151]: osdmap e116: 8 total, 8 up, 8 in 2024-04-26T16:24:45.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:24:45.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:45.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:45.274 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[86348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:24:45.274 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[86348]: osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033] boot 2024-04-26T16:24:45.274 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[86348]: osdmap e116: 8 total, 8 up, 8 in 2024-04-26T16:24:45.274 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:24:45.274 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:45.274 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:45.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:44 smithi064 ceph-mon[65271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:24:45.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:44 smithi064 ceph-mon[65271]: osd.1 [v2:172.21.15.59:6810/1197690033,v1:172.21.15.59:6811/1197690033] boot 2024-04-26T16:24:45.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:44 smithi064 ceph-mon[65271]: osdmap e116: 8 total, 8 up, 8 in 2024-04-26T16:24:45.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T16:24:45.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:45.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:46.241 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:45 smithi059 ceph-mon[81151]: pgmap v103: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 130 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:46.241 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:45 smithi059 ceph-mon[81151]: osdmap e117: 8 total, 8 up, 8 in 2024-04-26T16:24:46.241 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:45 smithi059 ceph-mon[86348]: pgmap v103: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 130 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:46.241 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:45 smithi059 ceph-mon[86348]: osdmap e117: 8 total, 8 up, 8 in 2024-04-26T16:24:46.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:45 smithi064 ceph-mon[65271]: pgmap v103: 161 pgs: 21 active+undersized+degraded, 42 active+undersized, 98 active+clean; 585 KiB data, 130 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-04-26T16:24:46.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:45 smithi064 ceph-mon[65271]: osdmap e117: 8 total, 8 up, 8 in 2024-04-26T16:24:47.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:47 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:47.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:47 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:47.552 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:47 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:47.552 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:47 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:47.553 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:47 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:47.553 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:47 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:24:48.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[86348]: pgmap v105: 161 pgs: 20 active+undersized+degraded, 37 active+undersized, 104 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 79/627 objects degraded (12.600%) 2024-04-26T16:24:48.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[86348]: Health check update: Degraded data redundancy: 79/627 objects degraded (12.600%), 20 pgs degraded (PG_DEGRADED) 2024-04-26T16:24:48.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:48.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:48.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:48.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:48.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:48.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:48.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:48.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[81151]: pgmap v105: 161 pgs: 20 active+undersized+degraded, 37 active+undersized, 104 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 79/627 objects degraded (12.600%) 2024-04-26T16:24:48.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[81151]: Health check update: Degraded data redundancy: 79/627 objects degraded (12.600%), 20 pgs degraded (PG_DEGRADED) 2024-04-26T16:24:48.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:48.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:48.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:48.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:48.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:48.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:48.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:48 smithi064 ceph-mon[65271]: pgmap v105: 161 pgs: 20 active+undersized+degraded, 37 active+undersized, 104 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 79/627 objects degraded (12.600%) 2024-04-26T16:24:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:48 smithi064 ceph-mon[65271]: Health check update: Degraded data redundancy: 79/627 objects degraded (12.600%), 20 pgs degraded (PG_DEGRADED) 2024-04-26T16:24:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:49.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-mon[81151]: pgmap v106: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s; 32/627 objects degraded (5.104%) 2024-04-26T16:24:49.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-mon[81151]: pgmap v107: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 32/627 objects degraded (5.104%) 2024-04-26T16:24:49.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:49.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:49.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-mon[86348]: pgmap v106: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s; 32/627 objects degraded (5.104%) 2024-04-26T16:24:49.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-mon[86348]: pgmap v107: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 32/627 objects degraded (5.104%) 2024-04-26T16:24:49.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:49.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:49.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:49 smithi064 ceph-mon[65271]: pgmap v106: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s; 32/627 objects degraded (5.104%) 2024-04-26T16:24:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:49 smithi064 ceph-mon[65271]: pgmap v107: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 32/627 objects degraded (5.104%) 2024-04-26T16:24:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:49 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:49.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:49 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:50.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:24:49.771+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:50.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:24:50.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:50.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:50.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:50.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:50.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:50.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:50.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:50.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:50.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:50.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:50.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:50.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:50.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:50.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:50.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:50.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:50.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:24:50.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:50.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:24:50.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:50.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:24:50.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:50.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:24:50.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:50.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:50.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:50.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:24:50.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:50.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:50.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:50.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:50.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:50.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:50.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:50.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:24:50.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:50.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:24:50.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:50.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:50.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:50.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:50.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:50.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:50.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:50.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:50.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:49 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:50.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:50.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:50.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:24:50.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:24:50.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:24:50.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:24:50.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:24:50.439 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.440 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:24:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:24:50.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:24:50.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:24:50.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:24:50.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:24:50.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:24:50.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:24:50.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:24:50.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.587 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:24:50.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:24:50.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:50.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:24:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:24:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:24:50.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:24:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:24:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:51.422 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:51.422 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:51.423 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:51.424 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:51.425 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:51.426 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:51.427 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:51.427 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:51.427 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:51.427 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:51.427 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:51.427 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.427 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.427 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:51.427 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.428 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:51.429 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:51.430 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:51.431 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:51.431 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:51.431 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:51.431 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:51.431 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:51.431 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:51.431 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: Upgrade: Finalizing container_image settings 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: Upgrade: Complete! 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 854 B/s rd, 0 op/s 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:51.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 924 B/s rd, 0 op/s 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[81151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/627 objects degraded (5.104%), 7 pgs degraded) 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:51.433 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: Upgrade: Finalizing container_image settings 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: Upgrade: Complete! 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 854 B/s rd, 0 op/s 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 924 B/s rd, 0 op/s 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:51.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:51.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:51.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:51.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:51.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:51 smithi059 ceph-mon[86348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/627 objects degraded (5.104%), 7 pgs degraded) 2024-04-26T16:24:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:51.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:24:51.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:51.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:24:51.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: Upgrade: Finalizing container_image settings 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: Upgrade: Complete! 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 854 B/s rd, 0 op/s 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 924 B/s rd, 0 op/s 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:51.442 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:51 smithi064 ceph-mon[65271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/627 objects degraded (5.104%), 7 pgs degraded) 2024-04-26T16:24:52.419 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:52 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:52.419 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:52 smithi059 ceph-mon[81151]: from='client.54318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:52.419 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:52 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:52.419 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:52 smithi059 ceph-mon[86348]: from='client.54318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:52.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:52 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:52.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:52 smithi064 ceph-mon[65271]: from='client.54318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:52.850 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:24:53.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:24:53.006+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:53.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:24:53.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:53.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:53.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:53.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:53.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:53.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:53.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:53.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:24:53.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:53.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:24:53.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:53.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:53.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:53.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:53.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:53.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:53.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:53.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:53.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:53.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:53.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:53.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:53.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:53.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:53.425 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-mon[81151]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-26T16:24:53.425 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:53.425 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:53.426 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:53.426 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:53.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-mon[86348]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-26T16:24:53.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:53.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:53.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:53.426 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:53 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:53.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:53 smithi064 ceph-mon[65271]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-26T16:24:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:53 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:24:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:53 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:53 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:24:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:53 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:54.426 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:24:54] "GET /metrics HTTP/1.1" 200 38176 "" "Prometheus/2.43.0" 2024-04-26T16:24:54.427 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:54.427 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:54.427 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:54.428 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:54.428 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:54.428 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.428 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.428 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:54.428 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:54.429 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:54.429 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:54.429 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:54.429 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:54.429 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:54.429 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:54.430 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:54.430 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:54.430 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:54.430 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:54.430 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.430 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.431 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:54.431 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:54.432 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:54.432 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:54.432 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:54.432 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:54.433 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:54.433 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:54.433 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:24:54.433 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:54.433 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:24:54.433 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:54.434 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:24:54.434 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:54.434 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:54.434 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:54.434 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:54.434 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:54.435 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:54.435 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:54.435 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:54.435 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:54.435 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.435 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.436 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:54.436 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:54.437 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:54.437 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:54.437 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:54.437 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:54.437 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:54.437 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:54.437 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:54.438 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:54.438 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:54.439 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:54.440 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:54.440 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:54.440 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:54.440 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:54.440 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:54.440 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:54.440 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:54.440 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:54.440 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:54.441 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:54.441 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.441 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.441 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:54.441 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:54.441 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:54.441 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:54.441 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:54.441 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:54.442 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:54.442 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:54.442 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:24:54.442 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:54.443 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:24:54.443 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:54.443 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:24:54.443 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:54.443 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:54.443 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:54.443 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:54.444 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:54.444 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:54.444 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:54.444 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:54.444 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:54.444 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.444 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.445 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:54.445 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:54.445 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:54.445 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:54.445 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:54.445 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:54.445 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:54.445 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:54.445 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[86348]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[86348]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:54.446 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[81151]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[81151]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:54.447 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:54.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:54 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:54.448 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:54 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:54.449 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:24:54.450 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:54.450 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:54.450 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:54.450 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:54.450 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:54.450 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:54.450 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:54.451 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.451 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:24:54.452 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:54.453 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:54.454 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:54.455 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:54.455 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:54 smithi064 ceph-mon[65271]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:24:54.455 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:54 smithi064 ceph-mon[65271]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:24:54.455 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:54 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:24:54.455 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:54 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:24:54.455 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:54 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:24:54.455 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:54 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:24:54.455 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:54 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:24:54.983 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (5m) 11s ago 27m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (5m) 2m ago 27m 80.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 11s ago 26m 62.2M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (6m) 2m ago 32m 461M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (16m) 11s ago 33m 520M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (3m) 11s ago 33m 50.7M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (3m) 2m ago 32m 40.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (2m) 11s ago 32m 41.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (6m) 11s ago 28m 16.5M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (6m) 2m ago 28m 17.2M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:24:54.984 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (53s) 11s ago 31m 51.3M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:24:54.985 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (16s) 11s ago 31m 12.5M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:24:54.985 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (31m) 11s ago 31m 53.6M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:24:54.985 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (30m) 11s ago 30m 53.8M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:24:54.985 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (30m) 2m ago 30m 55.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:24:54.985 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (29m) 2m ago 29m 54.5M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:24:54.985 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (29m) 2m ago 29m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:24:54.985 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (29m) 2m ago 29m 53.6M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:24:54.985 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (5m) 2m ago 28m 66.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:24:54.985 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (26m) 11s ago 26m 77.8M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:24:54.985 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (26m) 2m ago 26m 77.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:24:56.140 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-26T16:24:56.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:56 smithi064 ceph-mon[65271]: from='client.54324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:56.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:56 smithi064 ceph-mon[65271]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:24:56.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:56 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:56.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-mon[81151]: from='client.54324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:56.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-mon[81151]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:24:56.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-mon[86348]: from='client.54324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:24:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-mon[86348]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-26T16:24:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:24:56.942 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:24:56.725+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:56.942 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:24:56.942 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:56.942 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:56.942 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:56.942 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:56.942 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:56.942 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:56.942 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:56.942 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:24:56.943 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:56.943 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:24:56.943 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:56.943 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:56.943 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:56.943 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:56.943 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:56.943 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:56.943 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:56.944 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:56.944 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:56.944 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:56.944 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:56.944 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:56.944 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:56.944 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:56.944 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:56.944 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:56.944 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:56.945 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:56.946 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:56.947 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:56.947 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:56.947 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:56.947 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:24:56 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:58 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:58.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:58.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:58.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:58.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:58.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:24:58.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:58.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:24:58.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:58.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:24:58.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:58.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:58.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:58.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:58.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:58.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:58.338 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:58.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:58 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:58.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:58.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:58.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:58.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:58.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:58.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:24:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:24:58.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:58.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:58.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:58.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:24:58.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:58.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:58.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:58.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:58.343 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:58.343 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.343 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:58 smithi059 ceph-mon[81151]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 779 B/s rd, 0 op/s 2024-04-26T16:24:58.343 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:58.343 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:58.343 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:58.343 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.343 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:58 smithi059 ceph-mon[86348]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 779 B/s rd, 0 op/s 2024-04-26T16:24:58.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:58 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:58.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:58.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:58.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:58.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:58.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:58.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:58.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:24:58.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:24:58.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:24:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:24:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:24:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:24:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:24:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:24:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:24:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:24:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:24:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:24:58.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:58 smithi064 ceph-mon[65271]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 779 B/s rd, 0 op/s 2024-04-26T16:24:58.615 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:24:59.175 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:24:59 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/345641733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:59.175 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:24:59 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/345641733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:24:59.200 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-04-26T16:24:59.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:24:59 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/345641733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:00.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:00 smithi059 ceph-mon[86348]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:00.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:00 smithi059 ceph-mon[81151]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:00.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:00 smithi064 ceph-mon[65271]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:02.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:02 smithi064 ceph-mon[65271]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-26T16:25:02.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:02.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:02.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:02 smithi064 ceph-mon[65271]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:02.538 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:02 smithi059 ceph-mon[81151]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-26T16:25:02.538 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:02.538 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:02.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:02 smithi059 ceph-mon[81151]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:02.539 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:02 smithi059 ceph-mon[86348]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-26T16:25:02.539 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:02.539 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:02.539 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:02 smithi059 ceph-mon[86348]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:04.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:04 smithi064 ceph-mon[65271]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:04.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:04 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:04.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:25:04] "GET /metrics HTTP/1.1" 200 38315 "" "Prometheus/2.43.0" 2024-04-26T16:25:04.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:04 smithi059 ceph-mon[86348]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:04.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:04 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:04.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:04 smithi059 ceph-mon[81151]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:04.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:04 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:04.658 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:25:05.461 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade status' 2024-04-26T16:25:06.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:06 smithi064 ceph-mon[65271]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:06.531 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:06 smithi059 ceph-mon[81151]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:06.531 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:06 smithi059 ceph-mon[86348]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:07.561 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:25:07.562 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": null, 2024-04-26T16:25:07.562 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": false, 2024-04-26T16:25:07.562 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "", 2024-04-26T16:25:07.562 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:25:07.562 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": null, 2024-04-26T16:25:07.562 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "", 2024-04-26T16:25:07.562 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:25:07.562 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:25:08.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:08 smithi059 ceph-mon[81151]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:08.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:08 smithi059 ceph-mon[81151]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:08.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:08 smithi059 ceph-mon[86348]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:08.123 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:08 smithi059 ceph-mon[86348]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:08.147 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:25:08.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:08 smithi064 ceph-mon[65271]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:08.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:08 smithi064 ceph-mon[65271]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:10.243 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:25:10.243 INFO:teuthology.orchestra.run.smithi059.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:25:10.243 INFO:teuthology.orchestra.run.smithi059.stdout: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:25:10.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:10 smithi064 ceph-mon[65271]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:10.568 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:10 smithi059 ceph-mon[81151]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:10.569 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:10 smithi059 ceph-mon[86348]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:10.965 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-04-26T16:25:11.283 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:11 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/2152570794' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:25:11.283 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:11 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/2152570794' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:25:11.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:11 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2152570794' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:25:12.160 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:12 smithi059 ceph-mon[81151]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:12.160 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:12 smithi059 ceph-mon[86348]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:12.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:12 smithi064 ceph-mon[65271]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:14.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:14 smithi064 ceph-mon[65271]: from='client.54354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:14.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:14 smithi064 ceph-mon[65271]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:14.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:14 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:14 smithi059 ceph-mon[81151]: from='client.54354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:14 smithi059 ceph-mon[81151]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:14 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:14.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:25:14] "GET /metrics HTTP/1.1" 200 38315 "" "Prometheus/2.43.0" 2024-04-26T16:25:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:14 smithi059 ceph-mon[86348]: from='client.54354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:14 smithi059 ceph-mon[86348]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:14 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:14.997 INFO:teuthology.orchestra.run.smithi059.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:25:15.885 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-26T16:25:16.136 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:16.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:16.138 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:16.139 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:16.139 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:16 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:25:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-04-26T16:25:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:16.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:16.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:16.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:16.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:16.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:16 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:17.239 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:17.240 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:17.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:17 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:18.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:18 smithi059 ceph-mon[81151]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:18.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:18 smithi059 ceph-mon[86348]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:18.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:25:17.994+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:18.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:25:18.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:18.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:18.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:18.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:18.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:18.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:18.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:18.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:25:18.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:18.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:25:18.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:18.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:18.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:18.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:18.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:18.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:18.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:18.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:18.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:18.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:18.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:18.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:18.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:18.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:18.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:18.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:18.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:18.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:25:18.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:18.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:25:18.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:18.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:25:18.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:18.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:25:18.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:18.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:18.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:18.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:25:18.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:18.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:18.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:18.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:18.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:18.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:18.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:18.339 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:25:18.340 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:18.340 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:25:18.340 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:18.340 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:18.340 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:18.340 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:18.340 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:18.340 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:18.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:18.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:18.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:18.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:18 smithi064 ceph-mon[65271]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:19.185 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:25:19.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:19 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:19.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:19.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:19.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:19.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:19.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:25:19.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:19.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:19 smithi059 ceph-mon[81151]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:19 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:19.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:19.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:19 smithi059 ceph-mon[86348]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:25:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:19 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:19.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:19.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:19.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:19.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:19.441 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:19 smithi064 ceph-mon[65271]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:25:19.910 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (5m) 36s ago 28m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (5m) 3m ago 27m 80.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 36s ago 27m 62.2M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (7m) 3m ago 32m 461M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (17m) 36s ago 34m 520M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (3m) 36s ago 34m 50.7M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (4m) 3m ago 32m 40.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (3m) 36s ago 32m 41.1M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (6m) 36s ago 28m 16.5M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (6m) 3m ago 28m 17.2M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (78s) 36s ago 32m 51.3M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:25:19.911 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (41s) 36s ago 31m 12.5M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:25:19.912 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (31m) 36s ago 31m 53.6M 2993M 17.2.0 e1d6a67b021e b7e952887c22 2024-04-26T16:25:19.912 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (31m) 36s ago 31m 53.8M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:25:19.912 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (30m) 3m ago 30m 55.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:25:19.912 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (30m) 3m ago 30m 54.5M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:25:19.912 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (29m) 3m ago 29m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:25:19.912 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (29m) 3m ago 29m 53.6M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:25:19.912 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (6m) 3m ago 28m 66.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:25:19.913 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (27m) 36s ago 27m 77.8M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:25:19.913 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (27m) 3m ago 27m 77.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:25:20.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:20 smithi059 ceph-mon[81151]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:20.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:20 smithi059 ceph-mon[81151]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:20.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:20 smithi059 ceph-mon[81151]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:20.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:20 smithi059 ceph-mon[86348]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:20.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:20 smithi059 ceph-mon[86348]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:20.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:20 smithi059 ceph-mon[86348]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:20.340 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:25:20.341 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:25:20.341 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:25:20.341 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:25:20.341 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:25:20.341 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:25:20.342 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:25:20.342 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:25:20.342 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-26T16:25:20.342 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:25:20.342 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:25:20.342 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:25:20.343 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:25:20.343 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:25:20.343 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:25:20.343 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-26T16:25:20.343 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 7 2024-04-26T16:25:20.343 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:25:20.344 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:25:20.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:20 smithi064 ceph-mon[65271]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:20.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:20 smithi064 ceph-mon[65271]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:20.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:20 smithi064 ceph-mon[65271]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:20.702 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:25:20.702 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:25:20.702 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:25:20.702 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-04-26T16:25:20.702 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:25:20.702 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "2/8 daemons upgraded", 2024-04-26T16:25:20.702 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T16:25:20.702 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:25:20.702 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:25:21.271 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:21.271 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/4088512628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.271 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.271 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:25:21.271 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:25:21.271 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:21.271 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mgr 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mon 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all crash 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[86348]: Upgrade: osd.2 is safe to restart 2024-04-26T16:25:21.272 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/4088512628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mgr 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mon 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all crash 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.273 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T16:25:21.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T16:25:21.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:21 smithi059 ceph-mon[81151]: Upgrade: osd.2 is safe to restart 2024-04-26T16:25:21.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:21.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/4088512628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:25:21.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:25:21.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:21.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mgr 2024-04-26T16:25:21.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mon 2024-04-26T16:25:21.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:21.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all crash 2024-04-26T16:25:21.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:21.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T16:25:21.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T16:25:21.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:21 smithi064 ceph-mon[65271]: Upgrade: osd.2 is safe to restart 2024-04-26T16:25:22.070 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:22 smithi059 ceph-mon[81151]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:22.070 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:22 smithi059 ceph-mon[81151]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-26T16:25:22.070 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:22 smithi059 ceph-mon[86348]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:22.071 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:22 smithi059 ceph-mon[86348]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-26T16:25:22.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:22 smithi064 ceph-mon[65271]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:22.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:22 smithi064 ceph-mon[65271]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-26T16:25:23.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:23 smithi059 ceph-mon[81151]: Upgrade: Updating osd.2 2024-04-26T16:25:23.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:23.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T16:25:23.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:23.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:23 smithi059 ceph-mon[81151]: Deploying daemon osd.2 on smithi059 2024-04-26T16:25:23.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:23 smithi059 ceph-mon[86348]: Upgrade: Updating osd.2 2024-04-26T16:25:23.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:23.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T16:25:23.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:23.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:23 smithi059 ceph-mon[86348]: Deploying daemon osd.2 on smithi059 2024-04-26T16:25:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:23 smithi064 ceph-mon[65271]: Upgrade: Updating osd.2 2024-04-26T16:25:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T16:25:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:23 smithi064 ceph-mon[65271]: Deploying daemon osd.2 on smithi059 2024-04-26T16:25:24.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:24 smithi059 ceph-mon[81151]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:24.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:24 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:24.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:25:24] "GET /metrics HTTP/1.1" 200 38317 "" "Prometheus/2.43.0" 2024-04-26T16:25:24.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:24 smithi059 ceph-mon[86348]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:24.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:24 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:24.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:24 smithi064 ceph-mon[65271]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:24 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:25.616 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:25 smithi059 systemd[1]: Stopping Ceph osd.2 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:25:26.082 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[35313]: 2024-04-26T16:25:25.616+0000 7fda65a7d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:25:26.083 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[35313]: 2024-04-26T16:25:25.616+0000 7fda65a7d700 -1 osd.2 117 *** Got signal Terminated *** 2024-04-26T16:25:26.083 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[35313]: 2024-04-26T16:25:25.616+0000 7fda65a7d700 -1 osd.2 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:25:26.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:26 smithi064 ceph-mon[65271]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:26.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:26 smithi064 ceph-mon[65271]: osd.2 marked itself down and dead 2024-04-26T16:25:26.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:26 smithi059 ceph-mon[81151]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:26.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:26 smithi059 ceph-mon[81151]: osd.2 marked itself down and dead 2024-04-26T16:25:26.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:26 smithi059 ceph-mon[86348]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:26.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:26 smithi059 ceph-mon[86348]: osd.2 marked itself down and dead 2024-04-26T16:25:26.882 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:26 smithi059 podman[105873]: 2024-04-26 16:25:26.618732781 +0000 UTC m=+1.116387492 container died b7e952887c22969768009ac4f88c2af1dff0415f634c3fa8d864f9179692b59a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., version=8, io.openshift.tags=base centos centos-stream, release=754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, name=centos-stream, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, ceph=True) 2024-04-26T16:25:27.141 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:27 smithi059 ceph-mon[86348]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:25:27.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:27 smithi059 ceph-mon[86348]: osdmap e118: 8 total, 7 up, 8 in 2024-04-26T16:25:27.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:27 smithi059 ceph-mon[81151]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:25:27.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:27 smithi059 ceph-mon[81151]: osdmap e118: 8 total, 7 up, 8 in 2024-04-26T16:25:27.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:27 smithi064 ceph-mon[65271]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:25:27.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:27 smithi064 ceph-mon[65271]: osdmap e118: 8 total, 7 up, 8 in 2024-04-26T16:25:27.583 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:27 smithi059 podman[105873]: 2024-04-26 16:25:27.142126207 +0000 UTC m=+1.639780930 container cleanup b7e952887c22969768009ac4f88c2af1dff0415f634c3fa8d864f9179692b59a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, vendor=Red Hat, Inc., release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T16:25:27.583 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:27 smithi059 bash[105873]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2 2024-04-26T16:25:27.583 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:27 smithi059 podman[105890]: 2024-04-26 16:25:27.358997052 +0000 UTC m=+0.734715490 container remove b7e952887c22969768009ac4f88c2af1dff0415f634c3fa8d864f9179692b59a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2, release=754, io.openshift.tags=base centos centos-stream, vcs-type=git, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-26T16:25:28.026 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:27 smithi059 podman[105982]: 2024-04-26 16:25:27.675294182 +0000 UTC m=+0.018302233 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:25:28.026 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:27 smithi059 podman[105982]: 2024-04-26 16:25:27.809938789 +0000 UTC m=+0.152946863 container create c55c97a19a7aa66b1d2029c01ad60ace1e67d701c2d21a1f24cc7e27628619dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-26T16:25:28.026 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:28 smithi059 podman[105982]: 2024-04-26 16:25:28.01265714 +0000 UTC m=+0.355665182 container init c55c97a19a7aa66b1d2029c01ad60ace1e67d701c2d21a1f24cc7e27628619dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:25:28.027 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:28 smithi059 podman[105982]: 2024-04-26 16:25:28.016223021 +0000 UTC m=+0.359231054 container start c55c97a19a7aa66b1d2029c01ad60ace1e67d701c2d21a1f24cc7e27628619dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:25:28.331 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:28 smithi059 ceph-mon[81151]: pgmap v133: 161 pgs: 34 peering, 4 stale+active+clean, 123 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:28.331 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:28 smithi059 ceph-mon[81151]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-26T16:25:28.331 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:28 smithi059 ceph-mon[81151]: osdmap e119: 8 total, 7 up, 8 in 2024-04-26T16:25:28.332 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:28 smithi059 podman[105982]: 2024-04-26 16:25:28.112489502 +0000 UTC m=+0.455497555 container attach c55c97a19a7aa66b1d2029c01ad60ace1e67d701c2d21a1f24cc7e27628619dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0) 2024-04-26T16:25:28.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:28 smithi059 ceph-mon[86348]: pgmap v133: 161 pgs: 34 peering, 4 stale+active+clean, 123 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:28.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:28 smithi059 ceph-mon[86348]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-26T16:25:28.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:28 smithi059 ceph-mon[86348]: osdmap e119: 8 total, 7 up, 8 in 2024-04-26T16:25:28.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:28 smithi064 ceph-mon[65271]: pgmap v133: 161 pgs: 34 peering, 4 stale+active+clean, 123 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:28.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:28 smithi064 ceph-mon[65271]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-26T16:25:28.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:28 smithi064 ceph-mon[65271]: osdmap e119: 8 total, 7 up, 8 in 2024-04-26T16:25:28.582 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:28 smithi059 podman[105982]: 2024-04-26 16:25:28.332618498 +0000 UTC m=+0.675626528 container died c55c97a19a7aa66b1d2029c01ad60ace1e67d701c2d21a1f24cc7e27628619dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T16:25:29.082 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:28 smithi059 podman[106025]: 2024-04-26 16:25:28.983292733 +0000 UTC m=+0.644689865 container remove c55c97a19a7aa66b1d2029c01ad60ace1e67d701c2d21a1f24cc7e27628619dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-26T16:25:29.083 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:28 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-26T16:25:29.083 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:28 smithi059 systemd[1]: Stopped Ceph osd.2 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:25:29.083 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:28 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.2.service: Consumed 13.568s CPU time. 2024-04-26T16:25:29.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:29 smithi064 ceph-mon[65271]: Health check failed: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T16:25:29.452 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:29 smithi059 ceph-mon[81151]: Health check failed: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T16:25:29.452 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:29 smithi059 ceph-mon[86348]: Health check failed: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T16:25:29.452 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:29 smithi059 systemd[1]: Starting Ceph osd.2 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:25:29.703 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:29 smithi059 podman[106129]: 2024-04-26 16:25:29.355562714 +0000 UTC m=+0.018983074 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:25:29.703 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:29 smithi059 podman[106129]: 2024-04-26 16:25:29.492605384 +0000 UTC m=+0.156025776 container create 7bb3e73d8d88eae223ad5d1e5d0e93d93aaa5608151d8066b6576c6e1c9a12a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:25:30.082 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:29 smithi059 podman[106129]: 2024-04-26 16:25:29.7039322 +0000 UTC m=+0.367352560 container init 7bb3e73d8d88eae223ad5d1e5d0e93d93aaa5608151d8066b6576c6e1c9a12a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-26T16:25:30.083 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:29 smithi059 podman[106129]: 2024-04-26 16:25:29.70762844 +0000 UTC m=+0.371048791 container start 7bb3e73d8d88eae223ad5d1e5d0e93d93aaa5608151d8066b6576c6e1c9a12a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:25:30.083 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:29 smithi059 podman[106129]: 2024-04-26 16:25:29.786724414 +0000 UTC m=+0.450144773 container attach 7bb3e73d8d88eae223ad5d1e5d0e93d93aaa5608151d8066b6576c6e1c9a12a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-26T16:25:30.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:30 smithi064 ceph-mon[65271]: pgmap v135: 161 pgs: 2 active+undersized+degraded, 34 peering, 2 stale+active+clean, 5 active+undersized, 118 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 19/627 objects degraded (3.030%) 2024-04-26T16:25:30.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-mon[81151]: pgmap v135: 161 pgs: 2 active+undersized+degraded, 34 peering, 2 stale+active+clean, 5 active+undersized, 118 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 19/627 objects degraded (3.030%) 2024-04-26T16:25:30.514 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-mon[86348]: pgmap v135: 161 pgs: 2 active+undersized+degraded, 34 peering, 2 stale+active+clean, 5 active+undersized, 118 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 19/627 objects degraded (3.030%) 2024-04-26T16:25:30.514 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate[106144]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:25:30.514 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate[106144]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:25:30.514 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 bash[106129]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:25:30.514 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 bash[106129]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 bash[106129]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 bash[106129]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 bash[106129]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 bash[106129]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 bash[106129]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 bash[106129]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 bash[106129]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate[106144]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate[106144]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate[106144]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate[106144]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate[106144]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T16:25:30.515 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate[106144]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T16:25:30.516 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate[106144]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T16:25:30.516 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 podman[106129]: 2024-04-26 16:25:30.231606424 +0000 UTC m=+0.895026777 container died 7bb3e73d8d88eae223ad5d1e5d0e93d93aaa5608151d8066b6576c6e1c9a12a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:25:31.268 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:30 smithi059 podman[106279]: 2024-04-26 16:25:30.899595949 +0000 UTC m=+0.662677579 container remove 7bb3e73d8d88eae223ad5d1e5d0e93d93aaa5608151d8066b6576c6e1c9a12a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-26T16:25:31.268 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:31 smithi059 podman[106345]: 2024-04-26 16:25:31.023285797 +0000 UTC m=+0.018740682 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:25:31.268 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:31 smithi059 podman[106345]: 2024-04-26 16:25:31.133994826 +0000 UTC m=+0.129449741 container create e0a6538d233161d4bcdee4096403d0560714e3933c65689c07bdc34394a93f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:25:31.582 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:31 smithi059 podman[106345]: 2024-04-26 16:25:31.386659992 +0000 UTC m=+0.382114870 container init e0a6538d233161d4bcdee4096403d0560714e3933c65689c07bdc34394a93f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T16:25:31.583 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:31 smithi059 podman[106345]: 2024-04-26 16:25:31.390119078 +0000 UTC m=+0.385573950 container start e0a6538d233161d4bcdee4096403d0560714e3933c65689c07bdc34394a93f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-26T16:25:31.583 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:31 smithi059 ceph-osd[106364]: -- 172.21.15.59:0/3492535310 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a2f43b7680 con 0x55a2f4380c00 2024-04-26T16:25:31.583 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:31 smithi059 bash[106345]: e0a6538d233161d4bcdee4096403d0560714e3933c65689c07bdc34394a93f1a 2024-04-26T16:25:31.583 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:31 smithi059 systemd[1]: Started Ceph osd.2 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:25:32.332 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:31 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[106360]: 2024-04-26T16:25:31.937+0000 7f7900f487c0 -1 Falling back to public interface 2024-04-26T16:25:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[81151]: pgmap v136: 161 pgs: 3 active+undersized+degraded, 34 peering, 11 active+undersized, 113 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 24/627 objects degraded (3.828%) 2024-04-26T16:25:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:32.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[86348]: pgmap v136: 161 pgs: 3 active+undersized+degraded, 34 peering, 11 active+undersized, 113 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 24/627 objects degraded (3.828%) 2024-04-26T16:25:32.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:32.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:32.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:32 smithi064 ceph-mon[65271]: pgmap v136: 161 pgs: 3 active+undersized+degraded, 34 peering, 11 active+undersized, 113 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 24/627 objects degraded (3.828%) 2024-04-26T16:25:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:33.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:33 smithi059 ceph-mon[81151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-04-26T16:25:33.295 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:33 smithi059 ceph-mon[86348]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-04-26T16:25:33.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:33 smithi064 ceph-mon[65271]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-04-26T16:25:34.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:34 smithi064 ceph-mon[65271]: pgmap v137: 161 pgs: 16 active+undersized+degraded, 32 active+undersized, 113 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 58/627 objects degraded (9.250%) 2024-04-26T16:25:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:34 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:34.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:25:34] "GET /metrics HTTP/1.1" 200 38314 "" "Prometheus/2.43.0" 2024-04-26T16:25:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:34 smithi059 ceph-mon[86348]: pgmap v137: 161 pgs: 16 active+undersized+degraded, 32 active+undersized, 113 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 58/627 objects degraded (9.250%) 2024-04-26T16:25:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:34 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:34.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:34 smithi059 ceph-mon[81151]: pgmap v137: 161 pgs: 16 active+undersized+degraded, 32 active+undersized, 113 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 58/627 objects degraded (9.250%) 2024-04-26T16:25:34.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:34 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:34.950 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[106360]: 2024-04-26T16:25:34.668+0000 7f7900f487c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-26T16:25:34.950 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[106360]: 2024-04-26T16:25:34.690+0000 7f7900f487c0 -1 osd.2 117 log_to_monitors true 2024-04-26T16:25:35.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:35 smithi059 ceph-mon[81151]: from='osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T16:25:35.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:35 smithi059 ceph-mon[86348]: from='osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T16:25:35.333 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:25:35 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[106360]: 2024-04-26T16:25:35.180+0000 7f78f23a0700 -1 osd.2 117 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T16:25:35.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:35 smithi064 ceph-mon[65271]: from='osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T16:25:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:36 smithi064 ceph-mon[65271]: pgmap v138: 161 pgs: 16 active+undersized+degraded, 32 active+undersized, 113 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 58/627 objects degraded (9.250%) 2024-04-26T16:25:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:36 smithi064 ceph-mon[65271]: from='osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T16:25:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:36 smithi064 ceph-mon[65271]: osdmap e120: 8 total, 7 up, 8 in 2024-04-26T16:25:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:36 smithi064 ceph-mon[65271]: from='osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:25:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:36 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:36 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:36.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[81151]: pgmap v138: 161 pgs: 16 active+undersized+degraded, 32 active+undersized, 113 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 58/627 objects degraded (9.250%) 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[81151]: from='osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[81151]: osdmap e120: 8 total, 7 up, 8 in 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[81151]: from='osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[86348]: pgmap v138: 161 pgs: 16 active+undersized+degraded, 32 active+undersized, 113 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 58/627 objects degraded (9.250%) 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[86348]: from='osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[86348]: osdmap e120: 8 total, 7 up, 8 in 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[86348]: from='osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:36.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:36 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:37 smithi064 ceph-mon[65271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:25:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:37 smithi064 ceph-mon[65271]: osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830] boot 2024-04-26T16:25:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:37 smithi064 ceph-mon[65271]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T16:25:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:25:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:37 smithi064 ceph-mon[65271]: Health check update: Degraded data redundancy: 58/627 objects degraded (9.250%), 16 pgs degraded (PG_DEGRADED) 2024-04-26T16:25:37.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:37 smithi064 ceph-mon[65271]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T16:25:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[86348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:25:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[86348]: osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830] boot 2024-04-26T16:25:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[86348]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T16:25:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:25:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[86348]: Health check update: Degraded data redundancy: 58/627 objects degraded (9.250%), 16 pgs degraded (PG_DEGRADED) 2024-04-26T16:25:37.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[86348]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T16:25:37.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[81151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:25:37.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[81151]: osd.2 [v2:172.21.15.59:6818/2411229830,v1:172.21.15.59:6819/2411229830] boot 2024-04-26T16:25:37.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[81151]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T16:25:37.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T16:25:37.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:37.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:37.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[81151]: Health check update: Degraded data redundancy: 58/627 objects degraded (9.250%), 16 pgs degraded (PG_DEGRADED) 2024-04-26T16:25:37.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:37 smithi059 ceph-mon[81151]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T16:25:38.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:38 smithi059 ceph-mon[81151]: pgmap v141: 161 pgs: 26 peering, 3 active+undersized+degraded, 19 active+undersized, 113 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 24/627 objects degraded (3.828%) 2024-04-26T16:25:38.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:38 smithi059 ceph-mon[86348]: pgmap v141: 161 pgs: 26 peering, 3 active+undersized+degraded, 19 active+undersized, 113 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 24/627 objects degraded (3.828%) 2024-04-26T16:25:38.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:38 smithi064 ceph-mon[65271]: pgmap v141: 161 pgs: 26 peering, 3 active+undersized+degraded, 19 active+undersized, 113 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 24/627 objects degraded (3.828%) 2024-04-26T16:25:40.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:40 smithi059 ceph-mon[86348]: pgmap v143: 161 pgs: 26 peering, 1 active+undersized+degraded, 16 active+undersized, 118 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-26T16:25:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:40 smithi059 ceph-mon[81151]: pgmap v143: 161 pgs: 26 peering, 1 active+undersized+degraded, 16 active+undersized, 118 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-26T16:25:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:40 smithi064 ceph-mon[65271]: pgmap v143: 161 pgs: 26 peering, 1 active+undersized+degraded, 16 active+undersized, 118 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-26T16:25:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: pgmap v144: 161 pgs: 26 peering, 1 active+undersized+degraded, 4 active+undersized, 130 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 885 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-26T16:25:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: pgmap v145: 161 pgs: 26 peering, 1 active+undersized+degraded, 4 active+undersized, 130 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-26T16:25:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:41.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:41.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:41.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:41.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:41.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:41.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:41.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:41.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: pgmap v144: 161 pgs: 26 peering, 1 active+undersized+degraded, 4 active+undersized, 130 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 885 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-26T16:25:41.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: pgmap v145: 161 pgs: 26 peering, 1 active+undersized+degraded, 4 active+undersized, 130 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-26T16:25:41.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:41.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:41.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:41.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:41.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:41 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:42.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:42.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:42.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:42.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:42.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: pgmap v144: 161 pgs: 26 peering, 1 active+undersized+degraded, 4 active+undersized, 130 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 885 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-26T16:25:42.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: pgmap v145: 161 pgs: 26 peering, 1 active+undersized+degraded, 4 active+undersized, 130 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-26T16:25:42.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:42.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:42.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:42.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:42.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:41 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: Health check update: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded (PG_DEGRADED) 2024-04-26T16:25:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:25:43.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:25:42.730+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:43.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:43.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: Health check update: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded (PG_DEGRADED) 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:25:43.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:25:43.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:43.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:43.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:43.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:25:43.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:43.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:25:43.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:43.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:25:43.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:43.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:25:43.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:43.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:43.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:43.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:25:43.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:43.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:43.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:43.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:43.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:25:43.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:43.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:43.094 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:43.095 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:42 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:43.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: Health check update: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded (PG_DEGRADED) 2024-04-26T16:25:43.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:43.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:43.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:25:43.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:25:43.189 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:25:43.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:43.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:43.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:43.190 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:42 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:44.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:44.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:44.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:44.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:44.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:44.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:44.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:44.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:44.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:44.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:44.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:44.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:44.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:44.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:44.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:44.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:44.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:44.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:44.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:44.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:44.095 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:44.095 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:44.095 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.095 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.095 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:44.095 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:44.095 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:44.095 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:44.095 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:44.096 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:44.096 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:44.096 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:44.096 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:44.096 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:44.096 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:44.096 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:44.096 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.096 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: Upgrade: Finalizing container_image settings 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: Upgrade: Complete! 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 890 B/s rd, 0 op/s 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[86348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded) 2024-04-26T16:25:44.097 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:44.098 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:44.098 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:44.098 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.098 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: Upgrade: Finalizing container_image settings 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: Upgrade: Complete! 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 890 B/s rd, 0 op/s 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:44.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:43 smithi059 ceph-mon[81151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded) 2024-04-26T16:25:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:44.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:44.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:44.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: Upgrade: Finalizing container_image settings 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: Upgrade: Complete! 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-26T16:25:44.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 890 B/s rd, 0 op/s 2024-04-26T16:25:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:44.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:43 smithi064 ceph-mon[65271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded) 2024-04-26T16:25:44.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:25:44] "GET /metrics HTTP/1.1" 200 38314 "" "Prometheus/2.43.0" 2024-04-26T16:25:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:45.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:45.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:45.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:25:44.791+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:45.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:25:45.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:45.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:45.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:45.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:45.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:45.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:45.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:25:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:25:45.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[86348]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 671 B/s rd, 0 op/s 2024-04-26T16:25:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:46.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:46.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:46.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:46.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:46.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:46.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:46.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:46.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:46.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:46.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:46.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:46.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:46.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:46.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[81151]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 671 B/s rd, 0 op/s 2024-04-26T16:25:46.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:46.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:46.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:46.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:46.092 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[86348]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 0 op/s 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[86348]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:46.093 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:46.094 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.094 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[81151]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 0 op/s 2024-04-26T16:25:46.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[81151]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:25:46.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:46.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:46.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:46.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:45 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:45 smithi064 ceph-mon[65271]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 671 B/s rd, 0 op/s 2024-04-26T16:25:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:45 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:46.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:25:46.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:46.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:46.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:46.192 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:46.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:45 smithi064 ceph-mon[65271]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 0 op/s 2024-04-26T16:25:46.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:45 smithi064 ceph-mon[65271]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-26T16:25:46.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:45 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:25:46.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:25:46.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:25:46.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:45 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:25:47.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:25:46.806+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:47.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:25:47.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:47.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:47.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:47.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:47.083 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:47.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:47.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:47.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:47.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:47.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:47.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:47.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:47.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:47.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:47.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:47.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:47.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:47.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:47.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:47.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:47.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:47.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:47 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:47.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:47 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:47.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:47 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:47.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:47 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:47.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:47 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:47.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:47 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:47 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:47 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:25:47.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:47 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:48 smithi064 ceph-mon[65271]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 856 B/s rd, 0 op/s 2024-04-26T16:25:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:48 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.437 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.438 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:25:48.439 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:48.440 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:25:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.441 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:48 smithi059 ceph-mon[81151]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 856 B/s rd, 0 op/s 2024-04-26T16:25:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:48 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.583 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:48.584 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:25:48.585 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:48.586 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:48.587 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:48.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:48.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:48.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:48.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.588 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:48 smithi059 ceph-mon[86348]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 856 B/s rd, 0 op/s 2024-04-26T16:25:48.588 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:48 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:48.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:48.588 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:48.589 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:25:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:25:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:25:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:25:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:25:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:25:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:25:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:25:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:25:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:25:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:25:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:25:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:25:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:25:50.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:50 smithi064 ceph-mon[65271]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-04-26T16:25:50.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:50 smithi059 ceph-mon[81151]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-04-26T16:25:50.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:50 smithi059 ceph-mon[86348]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-04-26T16:25:51.872 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:25:52.420 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:52 smithi059 ceph-mon[81151]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-26T16:25:52.420 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:52 smithi059 ceph-mon[81151]: from='client.54396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:52.420 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:52 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:52.420 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:52 smithi059 ceph-mon[86348]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-26T16:25:52.420 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:52 smithi059 ceph-mon[86348]: from='client.54396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:52.420 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:52 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:52.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:52 smithi064 ceph-mon[65271]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-26T16:25:52.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:52 smithi064 ceph-mon[65271]: from='client.54396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:52.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:52 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:25:53.993 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:25:53.993 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (6m) 18s ago 28m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:25:53.993 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (6m) 3m ago 28m 80.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:25:53.993 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 18s ago 27m 62.3M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:25:53.993 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (7m) 3m ago 33m 461M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:25:53.993 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (17m) 18s ago 34m 522M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (4m) 18s ago 34m 54.4M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (4m) 3m ago 33m 40.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (3m) 18s ago 33m 44.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (7m) 18s ago 29m 16.7M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (7m) 3m ago 29m 17.2M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (112s) 18s ago 32m 51.4M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (75s) 18s ago 32m 43.4M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (22s) 18s ago 32m 12.7M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (31m) 18s ago 31m 55.1M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (31m) 3m ago 31m 55.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (30m) 3m ago 30m 54.5M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (30m) 3m ago 30m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:25:53.994 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (30m) 3m ago 30m 53.6M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:25:53.995 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (6m) 3m ago 29m 66.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:25:53.995 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (27m) 18s ago 27m 78.1M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:25:53.995 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (27m) 3m ago 27m 77.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:25:54.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:25:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:25:54] "GET /metrics HTTP/1.1" 200 38329 "" "Prometheus/2.43.0" 2024-04-26T16:25:54.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:54 smithi059 ceph-mon[81151]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:54.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:54 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:54.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:54 smithi059 ceph-mon[86348]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:54.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:54 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:54.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:54 smithi064 ceph-mon[65271]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:25:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:54 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:25:54.642 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-26T16:25:55.259 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:55 smithi059 ceph-mon[81151]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:55.260 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:55 smithi059 ceph-mon[86348]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:55.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:55 smithi064 ceph-mon[65271]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:25:56.281 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:56 smithi059 ceph-mon[81151]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:56.281 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:56 smithi059 ceph-mon[86348]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:56.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:56 smithi064 ceph-mon[65271]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:25:56.945 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:25:57.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:57 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/3532677929' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:57.546 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:57 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/3532677929' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:57.546 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:57 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/3532677929' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:25:57.570 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-04-26T16:25:58.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:25:58 smithi064 ceph-mon[65271]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:58.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:25:58 smithi059 ceph-mon[81151]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:25:58.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:25:58 smithi059 ceph-mon[86348]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:00.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:00 smithi064 ceph-mon[65271]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:00.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:00 smithi059 ceph-mon[86348]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:00.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:00 smithi059 ceph-mon[81151]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:01.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:01 smithi059 ceph-mon[81151]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:01.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:01 smithi059 ceph-mon[86348]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:01.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:01.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:01 smithi064 ceph-mon[65271]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:01.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:01.749 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:26:02.486 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:02 smithi059 ceph-mon[81151]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:02.487 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:02 smithi059 ceph-mon[86348]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:02.514 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade status' 2024-04-26T16:26:02.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:02 smithi064 ceph-mon[65271]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:04.296 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:26:04] "GET /metrics HTTP/1.1" 200 38326 "" "Prometheus/2.43.0" 2024-04-26T16:26:04.296 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:04 smithi059 ceph-mon[86348]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:04.297 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:04 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:04.297 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:04 smithi059 ceph-mon[81151]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:04.297 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:04 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:04.466 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:26:04.466 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": null, 2024-04-26T16:26:04.466 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": false, 2024-04-26T16:26:04.466 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "", 2024-04-26T16:26:04.466 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:26:04.466 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": null, 2024-04-26T16:26:04.467 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "", 2024-04-26T16:26:04.467 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:26:04.467 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:26:04.489 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:04 smithi064 ceph-mon[65271]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:04.489 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:04 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:05.087 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:26:05.536 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:05 smithi059 ceph-mon[81151]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:05.536 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:05 smithi059 ceph-mon[86348]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:05.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:05 smithi064 ceph-mon[65271]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:06.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:06 smithi059 ceph-mon[81151]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:06.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:06 smithi059 ceph-mon[86348]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:06.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:06 smithi064 ceph-mon[65271]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:07.162 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:26:07.163 INFO:teuthology.orchestra.run.smithi059.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:26:07.163 INFO:teuthology.orchestra.run.smithi059.stdout: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:26:07.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:07 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/936857555' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:26:07.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:07 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/936857555' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:26:07.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:07 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/936857555' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:26:07.852 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-04-26T16:26:08.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:08 smithi059 ceph-mon[86348]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:08.449 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:08 smithi059 ceph-mon[81151]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:08.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:08 smithi064 ceph-mon[65271]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:10.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:10 smithi059 ceph-mon[81151]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:10.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:10 smithi059 ceph-mon[86348]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:10.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:10 smithi064 ceph-mon[65271]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:11.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:11 smithi059 ceph-mon[86348]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:11 smithi059 ceph-mon[81151]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:11 smithi064 ceph-mon[65271]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:12.130 INFO:teuthology.orchestra.run.smithi059.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:26:12.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:12 smithi059 ceph-mon[81151]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:12.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:12.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:12.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:12 smithi059 ceph-mon[86348]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:12.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:12.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:12.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:12 smithi064 ceph-mon[65271]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:26:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:12.883 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-26T16:26:13.535 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[81151]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:26:13.535 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:13.536 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:26:13.536 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:13.536 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:26:13.536 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:13.536 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[86348]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:26:13.536 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:13.536 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:26:13.536 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:13.536 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:26:13.536 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:13 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:13 smithi064 ceph-mon[65271]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:26:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:13 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:13 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:26:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:13 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:13 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:26:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:13 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:14.407 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-mon[81151]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:14.408 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-mon[81151]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:14.408 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:26:14.408 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:26:14.408 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:14.408 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:26:14] "GET /metrics HTTP/1.1" 200 38326 "" "Prometheus/2.43.0" 2024-04-26T16:26:14.408 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-mon[86348]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:14.409 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-mon[86348]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:14.409 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:26:14.409 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:26:14.409 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:14 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:14.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:14 smithi064 ceph-mon[65271]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:26:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:14 smithi064 ceph-mon[65271]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:14 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:26:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:14 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:26:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:14 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:15.031 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:26:15.751 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:26:15.751 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (6m) 39s ago 28m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:26:15.751 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (6m) 4m ago 28m 80.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:26:15.751 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 39s ago 28m 62.3M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:26:15.751 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (8m) 4m ago 33m 461M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:26:15.751 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (18m) 39s ago 35m 522M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:26:15.751 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (4m) 39s ago 35m 54.4M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (5m) 4m ago 33m 40.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (3m) 39s ago 33m 44.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (7m) 39s ago 29m 16.7M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (7m) 4m ago 29m 17.2M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (2m) 39s ago 33m 51.4M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (97s) 39s ago 32m 43.4M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (44s) 39s ago 32m 12.7M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (31m) 39s ago 31m 55.1M 2993M 17.2.0 e1d6a67b021e 2f24d0976f65 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (31m) 4m ago 31m 55.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (31m) 4m ago 31m 54.5M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (30m) 4m ago 30m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (30m) 4m ago 30m 53.6M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (7m) 4m ago 29m 66.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (28m) 39s ago 28m 78.1M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:26:15.752 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (28m) 4m ago 28m 77.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:26:16.197 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:26:16.197 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:26:16.197 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:26:16.197 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:26:16.198 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:26:16.198 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:26:16.198 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:26:16.198 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:26:16.198 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-26T16:26:16.198 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:26:16.198 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:26:16.198 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:26:16.199 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:26:16.199 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:26:16.199 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:26:16.199 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-26T16:26:16.199 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T16:26:16.199 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:26:16.199 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:26:16.228 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:26:16.113+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:16.228 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:26:16.228 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:16.228 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:16.228 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:16.229 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:16.229 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:16.229 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:26:16.229 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:16.229 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:26:16.229 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:16.229 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:26:16.229 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:16.230 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:16.230 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:16.230 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:16.230 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:16.230 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:16.230 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:16.231 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:16.231 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:16.231 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:26:16.231 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:26:16.231 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:26:16.232 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:26:16.232 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:26:16.232 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:26:16.232 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:26:16.232 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:26:16.232 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:26:16.232 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:26:16.232 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:26:16.233 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:26:16.233 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:26:16.233 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:26:16.233 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:26:16.233 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:26:16.233 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:26:16.233 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:16.233 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:26:16.233 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:16.234 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:16.235 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:16.235 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:16.235 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:16.235 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:26:16.235 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:16.235 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:26:16.235 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:16.236 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:26:16.236 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:16.236 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:16.236 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:16.236 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:16.237 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:16.237 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:16.237 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:16.238 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:16.238 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:16.560 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:26:16.560 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", 2024-04-26T16:26:16.560 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:26:16.560 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-26T16:26:16.560 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:26:16.560 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "", 2024-04-26T16:26:16.560 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 image", 2024-04-26T16:26:16.560 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:26:16.560 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:26:16.561 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-mon[86348]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:16.561 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-mon[86348]: from='client.54438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:16.561 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-mon[86348]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:16.561 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:16.561 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/2267980150' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:16.561 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-mon[81151]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:16.561 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-mon[81151]: from='client.54438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:16.561 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-mon[81151]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:16.561 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:16.562 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:16 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/2267980150' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:16.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:16 smithi064 ceph-mon[65271]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:16.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:16 smithi064 ceph-mon[65271]: from='client.54438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:16.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:16 smithi064 ceph-mon[65271]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:16.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:16.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:16 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2267980150' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:17 smithi059 ceph-mon[86348]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:17 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:17.308 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:17.309 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:26:17.310 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:17.311 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:17.312 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:26:17.312 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:17.312 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:17.312 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:17.312 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:17.312 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:17 smithi059 ceph-mon[81151]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:17.312 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:17 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:17.312 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:17.312 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:17.312 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:17.313 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:26:17.314 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:17.315 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:17.316 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:17.317 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:17.317 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:17.317 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:17.317 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.317 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.317 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:17 smithi059 ceph-mon[86348]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:26:17.317 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:17 smithi059 ceph-mon[86348]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:17.317 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:17.317 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:17.317 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:17.318 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.318 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.318 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:17 smithi059 ceph-mon[81151]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:26:17.318 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:17 smithi059 ceph-mon[81151]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:17 smithi064 ceph-mon[65271]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:17 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:17.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:17.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:26:17.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:17.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:17.692 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:17.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:17 smithi064 ceph-mon[65271]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:26:17.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:17 smithi064 ceph-mon[65271]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[86348]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T16:26:18.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[81151]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:18.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:18.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:18.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:18.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:18.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:18 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T16:26:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:18 smithi064 ceph-mon[65271]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:18 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:18 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:18 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:18 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:18 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:18 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:18 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:18.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:18 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:18.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:18 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T16:26:19.433 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[86348]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:26:19.433 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[86348]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:26:19.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mgr 2024-04-26T16:26:19.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mon 2024-04-26T16:26:19.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all crash 2024-04-26T16:26:19.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T16:26:19.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[86348]: Upgrade: osd.3 is safe to restart 2024-04-26T16:26:19.434 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[81151]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:26:19.434 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[81151]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:26:19.434 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mgr 2024-04-26T16:26:19.434 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mon 2024-04-26T16:26:19.435 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all crash 2024-04-26T16:26:19.435 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T16:26:19.435 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:19 smithi059 ceph-mon[81151]: Upgrade: osd.3 is safe to restart 2024-04-26T16:26:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:19 smithi064 ceph-mon[65271]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:26:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:19 smithi064 ceph-mon[65271]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:26:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:19 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mgr 2024-04-26T16:26:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:19 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mon 2024-04-26T16:26:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:19 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all crash 2024-04-26T16:26:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:19 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T16:26:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:19 smithi064 ceph-mon[65271]: Upgrade: osd.3 is safe to restart 2024-04-26T16:26:20.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:20 smithi059 ceph-mon[81151]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:20.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:20 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:20.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:20 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T16:26:20.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:20 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:20.548 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:20 smithi059 ceph-mon[86348]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:20.548 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:20 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:20.548 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:20 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T16:26:20.548 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:20 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:20 smithi064 ceph-mon[65271]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:20 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:20 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T16:26:20.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:20 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:21.572 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:21 smithi059 ceph-mon[86348]: Upgrade: Updating osd.3 2024-04-26T16:26:21.572 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:21 smithi059 ceph-mon[86348]: Deploying daemon osd.3 on smithi059 2024-04-26T16:26:21.572 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:21 smithi059 ceph-mon[81151]: Upgrade: Updating osd.3 2024-04-26T16:26:21.573 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:21 smithi059 ceph-mon[81151]: Deploying daemon osd.3 on smithi059 2024-04-26T16:26:21.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:21 smithi064 ceph-mon[65271]: Upgrade: Updating osd.3 2024-04-26T16:26:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:21 smithi064 ceph-mon[65271]: Deploying daemon osd.3 on smithi059 2024-04-26T16:26:22.397 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:22 smithi059 ceph-mon[86348]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:22.397 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:22 smithi059 ceph-mon[81151]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:22.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:22 smithi064 ceph-mon[65271]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-26T16:26:23.083 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:22 smithi059 systemd[1]: Stopping Ceph osd.3 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:26:23.583 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[38311]: 2024-04-26T16:26:23.269+0000 7f4b96dd9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:26:23.583 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[38311]: 2024-04-26T16:26:23.269+0000 7f4b96dd9700 -1 osd.3 122 *** Got signal Terminated *** 2024-04-26T16:26:23.583 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[38311]: 2024-04-26T16:26:23.269+0000 7f4b96dd9700 -1 osd.3 122 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:26:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:24 smithi059 ceph-mon[81151]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:26:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:24 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:24.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:24 smithi059 ceph-mon[81151]: osd.3 marked itself down and dead 2024-04-26T16:26:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:24 smithi059 ceph-mon[86348]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:26:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:24 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:24.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:24 smithi059 ceph-mon[86348]: osd.3 marked itself down and dead 2024-04-26T16:26:24.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:26:24] "GET /metrics HTTP/1.1" 200 38325 "" "Prometheus/2.43.0" 2024-04-26T16:26:24.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:24 smithi064 ceph-mon[65271]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:26:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:24 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:24 smithi064 ceph-mon[65271]: osd.3 marked itself down and dead 2024-04-26T16:26:25.013 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:24 smithi059 podman[113029]: 2024-04-26 16:26:24.735547431 +0000 UTC m=+1.595052017 container died 2f24d0976f6565b378c7a71f1c0b04860e3c64feeb2a83c8c1bd20babdc5460c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, ceph=True, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, RELEASE=HEAD, io.buildah.version=1.19.8, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux ) 2024-04-26T16:26:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:25 smithi059 ceph-mon[86348]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:26:25.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:25 smithi059 ceph-mon[86348]: osdmap e123: 8 total, 7 up, 8 in 2024-04-26T16:26:25.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:25 smithi059 ceph-mon[81151]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:26:25.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:25 smithi059 ceph-mon[81151]: osdmap e123: 8 total, 7 up, 8 in 2024-04-26T16:26:25.333 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:25 smithi059 podman[113029]: 2024-04-26 16:26:25.187291567 +0000 UTC m=+2.046796152 container cleanup 2f24d0976f6565b378c7a71f1c0b04860e3c64feeb2a83c8c1bd20babdc5460c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, release=754, RELEASE=HEAD, io.openshift.expose-services=, com.redhat.component=centos-stream-container, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vendor=Red Hat, Inc., version=8) 2024-04-26T16:26:25.334 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:25 smithi059 bash[113029]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3 2024-04-26T16:26:25.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:25 smithi064 ceph-mon[65271]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:26:25.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:25 smithi064 ceph-mon[65271]: osdmap e123: 8 total, 7 up, 8 in 2024-04-26T16:26:25.809 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:25 smithi059 podman[113046]: 2024-04-26 16:26:25.380865875 +0000 UTC m=+0.638573307 container remove 2f24d0976f6565b378c7a71f1c0b04860e3c64feeb2a83c8c1bd20babdc5460c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, ceph=True, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git) 2024-04-26T16:26:26.066 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:25 smithi059 podman[113137]: 2024-04-26 16:26:25.713531863 +0000 UTC m=+0.018708718 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:26:26.067 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:25 smithi059 podman[113137]: 2024-04-26 16:26:25.814914807 +0000 UTC m=+0.120091650 container create cb871ee7cfe6a673e59f3f178d9a2e40c5d67221233c1f2f1e52a885a797e796 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-26T16:26:26.067 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:26 smithi059 podman[113137]: 2024-04-26 16:26:26.050835284 +0000 UTC m=+0.356012127 container init cb871ee7cfe6a673e59f3f178d9a2e40c5d67221233c1f2f1e52a885a797e796 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T16:26:26.067 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:26 smithi059 podman[113137]: 2024-04-26 16:26:26.054973926 +0000 UTC m=+0.360150776 container start cb871ee7cfe6a673e59f3f178d9a2e40c5d67221233c1f2f1e52a885a797e796 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:26:26.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:26 smithi059 ceph-mon[81151]: pgmap v172: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:26:26.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:26 smithi059 ceph-mon[81151]: osdmap e124: 8 total, 7 up, 8 in 2024-04-26T16:26:26.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:26 smithi059 ceph-mon[86348]: pgmap v172: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:26:26.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:26 smithi059 ceph-mon[86348]: osdmap e124: 8 total, 7 up, 8 in 2024-04-26T16:26:26.333 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:26 smithi059 podman[113137]: 2024-04-26 16:26:26.159190078 +0000 UTC m=+0.464366936 container attach cb871ee7cfe6a673e59f3f178d9a2e40c5d67221233c1f2f1e52a885a797e796 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:26:26.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:26 smithi064 ceph-mon[65271]: pgmap v172: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:26:26.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:26 smithi064 ceph-mon[65271]: osdmap e124: 8 total, 7 up, 8 in 2024-04-26T16:26:26.730 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:26 smithi059 podman[113137]: 2024-04-26 16:26:26.367993045 +0000 UTC m=+0.673169888 container died cb871ee7cfe6a673e59f3f178d9a2e40c5d67221233c1f2f1e52a885a797e796 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-26T16:26:27.324 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:27 smithi059 podman[113182]: 2024-04-26 16:26:27.063511199 +0000 UTC m=+0.689667072 container remove cb871ee7cfe6a673e59f3f178d9a2e40c5d67221233c1f2f1e52a885a797e796 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-26T16:26:27.325 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:27 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-26T16:26:27.325 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:27 smithi059 systemd[1]: Stopped Ceph osd.3 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:26:27.325 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:27 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.3.service: Consumed 17.750s CPU time. 2024-04-26T16:26:27.325 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:27 smithi059 systemd[1]: Starting Ceph osd.3 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:26:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:27 smithi059 ceph-mon[81151]: Health check failed: Degraded data redundancy: 23/627 objects degraded (3.668%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:27 smithi059 ceph-mon[86348]: Health check failed: Degraded data redundancy: 23/627 objects degraded (3.668%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:27.583 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:27 smithi059 podman[113279]: 2024-04-26 16:26:27.426276939 +0000 UTC m=+0.018850217 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:26:27.583 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:27 smithi059 podman[113279]: 2024-04-26 16:26:27.54780132 +0000 UTC m=+0.140374627 container create 5683f78ca81150c484b56c1a1ffe837dfe663a2e39fb1a566f5d10dcf87f1470 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:26:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:27 smithi064 ceph-mon[65271]: Health check failed: Degraded data redundancy: 23/627 objects degraded (3.668%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:28.083 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:27 smithi059 podman[113279]: 2024-04-26 16:26:27.766913108 +0000 UTC m=+0.359486373 container init 5683f78ca81150c484b56c1a1ffe837dfe663a2e39fb1a566f5d10dcf87f1470 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:26:28.083 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:27 smithi059 podman[113279]: 2024-04-26 16:26:27.770386899 +0000 UTC m=+0.362960166 container start 5683f78ca81150c484b56c1a1ffe837dfe663a2e39fb1a566f5d10dcf87f1470 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:26:28.083 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:27 smithi059 podman[113279]: 2024-04-26 16:26:27.841884193 +0000 UTC m=+0.434457468 container attach 5683f78ca81150c484b56c1a1ffe837dfe663a2e39fb1a566f5d10dcf87f1470 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424) 2024-04-26T16:26:28.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-mon[81151]: pgmap v174: 161 pgs: 6 active+undersized+degraded, 7 peering, 19 stale+active+clean, 8 active+undersized, 121 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 23/627 objects degraded (3.668%) 2024-04-26T16:26:28.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-mon[86348]: pgmap v174: 161 pgs: 6 active+undersized+degraded, 7 peering, 19 stale+active+clean, 8 active+undersized, 121 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 23/627 objects degraded (3.668%) 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate[113295]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate[113295]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 bash[113279]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 bash[113279]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 bash[113279]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 bash[113279]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 bash[113279]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 bash[113279]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 bash[113279]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 bash[113279]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 bash[113279]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate[113295]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T16:26:28.584 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate[113295]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T16:26:28.585 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate[113295]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T16:26:28.585 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate[113295]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T16:26:28.585 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate[113295]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T16:26:28.585 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate[113295]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T16:26:28.585 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate[113295]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T16:26:28.585 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:28 smithi059 podman[113279]: 2024-04-26 16:26:28.289581741 +0000 UTC m=+0.882155007 container died 5683f78ca81150c484b56c1a1ffe837dfe663a2e39fb1a566f5d10dcf87f1470 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:26:28.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:28 smithi064 ceph-mon[65271]: pgmap v174: 161 pgs: 6 active+undersized+degraded, 7 peering, 19 stale+active+clean, 8 active+undersized, 121 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 23/627 objects degraded (3.668%) 2024-04-26T16:26:29.333 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:29 smithi059 podman[113419]: 2024-04-26 16:26:29.038236959 +0000 UTC m=+0.743360933 container remove 5683f78ca81150c484b56c1a1ffe837dfe663a2e39fb1a566f5d10dcf87f1470 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T16:26:29.333 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:29 smithi059 podman[113486]: 2024-04-26 16:26:29.255872043 +0000 UTC m=+0.108053500 container create 0c13cc20260a5f1b489f6ce9fff32d6d6e463124a1f537567eaa16fe285541d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-26T16:26:29.333 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:29 smithi059 podman[113486]: 2024-04-26 16:26:29.166769948 +0000 UTC m=+0.018951402 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:26:29.833 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:29 smithi059 podman[113486]: 2024-04-26 16:26:29.52476513 +0000 UTC m=+0.376946585 container init 0c13cc20260a5f1b489f6ce9fff32d6d6e463124a1f537567eaa16fe285541d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T16:26:29.833 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:29 smithi059 podman[113486]: 2024-04-26 16:26:29.527955581 +0000 UTC m=+0.380137036 container start 0c13cc20260a5f1b489f6ce9fff32d6d6e463124a1f537567eaa16fe285541d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424) 2024-04-26T16:26:29.833 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:29 smithi059 ceph-osd[113505]: -- 172.21.15.59:0/4094763304 <== mon.1 v2:172.21.15.59:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562e9ae79680 con 0x562e9ae43000 2024-04-26T16:26:29.833 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:29 smithi059 bash[113486]: 0c13cc20260a5f1b489f6ce9fff32d6d6e463124a1f537567eaa16fe285541d3 2024-04-26T16:26:29.833 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:29 smithi059 systemd[1]: Started Ceph osd.3 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:26:30.330 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[113501]: 2024-04-26T16:26:30.071+0000 7f480a2b67c0 -1 Falling back to public interface 2024-04-26T16:26:30.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:30 smithi059 ceph-mon[86348]: pgmap v175: 161 pgs: 24 active+undersized+degraded, 7 peering, 35 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:30.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:30.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:30.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:30.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:30 smithi059 ceph-mon[81151]: pgmap v175: 161 pgs: 24 active+undersized+degraded, 7 peering, 35 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:30.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:30.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:30.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:30.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:30 smithi064 ceph-mon[65271]: pgmap v175: 161 pgs: 24 active+undersized+degraded, 7 peering, 35 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:32.326 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:32 smithi059 ceph-mon[86348]: pgmap v176: 161 pgs: 24 active+undersized+degraded, 42 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:32.326 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:32.326 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:32.326 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:32 smithi059 ceph-mon[81151]: pgmap v176: 161 pgs: 24 active+undersized+degraded, 42 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:32.326 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:32.326 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:32.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:32 smithi064 ceph-mon[65271]: pgmap v176: 161 pgs: 24 active+undersized+degraded, 42 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:33.037 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[113501]: 2024-04-26T16:26:32.817+0000 7f480a2b67c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-26T16:26:33.037 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:32 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[113501]: 2024-04-26T16:26:32.847+0000 7f480a2b67c0 -1 osd.3 122 log_to_monitors true 2024-04-26T16:26:33.298 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:33 smithi059 ceph-mon[81151]: from='osd.3 [v2:172.21.15.59:6826/3455081089,v1:172.21.15.59:6827/3455081089]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T16:26:33.298 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:33 smithi059 ceph-mon[81151]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T16:26:33.298 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:33 smithi059 ceph-mon[86348]: from='osd.3 [v2:172.21.15.59:6826/3455081089,v1:172.21.15.59:6827/3455081089]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T16:26:33.299 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:33 smithi059 ceph-mon[86348]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T16:26:33.299 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:26:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[113501]: 2024-04-26T16:26:33.124+0000 7f47fb70e700 -1 osd.3 122 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T16:26:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:33 smithi064 ceph-mon[65271]: from='osd.3 [v2:172.21.15.59:6826/3455081089,v1:172.21.15.59:6827/3455081089]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T16:26:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:33 smithi064 ceph-mon[65271]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T16:26:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:34 smithi064 ceph-mon[65271]: pgmap v177: 161 pgs: 24 active+undersized+degraded, 42 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:34 smithi064 ceph-mon[65271]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T16:26:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:34 smithi064 ceph-mon[65271]: osdmap e125: 8 total, 7 up, 8 in 2024-04-26T16:26:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:34 smithi064 ceph-mon[65271]: from='osd.3 [v2:172.21.15.59:6826/3455081089,v1:172.21.15.59:6827/3455081089]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:26:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:34 smithi064 ceph-mon[65271]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:26:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:34 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:34 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:34 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:34.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:26:34] "GET /metrics HTTP/1.1" 200 38326 "" "Prometheus/2.43.0" 2024-04-26T16:26:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[86348]: pgmap v177: 161 pgs: 24 active+undersized+degraded, 42 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[86348]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T16:26:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[86348]: osdmap e125: 8 total, 7 up, 8 in 2024-04-26T16:26:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[86348]: from='osd.3 [v2:172.21.15.59:6826/3455081089,v1:172.21.15.59:6827/3455081089]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:26:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[86348]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:26:34.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:34.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:34.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:34.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[81151]: pgmap v177: 161 pgs: 24 active+undersized+degraded, 42 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:34.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[81151]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T16:26:34.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[81151]: osdmap e125: 8 total, 7 up, 8 in 2024-04-26T16:26:34.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[81151]: from='osd.3 [v2:172.21.15.59:6826/3455081089,v1:172.21.15.59:6827/3455081089]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:26:34.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[81151]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2024-04-26T16:26:34.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:34.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:34.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:34 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:35 smithi064 ceph-mon[65271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:26:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:35 smithi064 ceph-mon[65271]: osd.3 [v2:172.21.15.59:6826/3455081089,v1:172.21.15.59:6827/3455081089] boot 2024-04-26T16:26:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:35 smithi064 ceph-mon[65271]: osdmap e126: 8 total, 8 up, 8 in 2024-04-26T16:26:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:26:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:35 smithi064 ceph-mon[65271]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T16:26:35.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:35 smithi059 ceph-mon[81151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:26:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:35 smithi059 ceph-mon[81151]: osd.3 [v2:172.21.15.59:6826/3455081089,v1:172.21.15.59:6827/3455081089] boot 2024-04-26T16:26:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:35 smithi059 ceph-mon[81151]: osdmap e126: 8 total, 8 up, 8 in 2024-04-26T16:26:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:26:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:35 smithi059 ceph-mon[81151]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T16:26:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:35 smithi059 ceph-mon[86348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:26:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:35 smithi059 ceph-mon[86348]: osd.3 [v2:172.21.15.59:6826/3455081089,v1:172.21.15.59:6827/3455081089] boot 2024-04-26T16:26:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:35 smithi059 ceph-mon[86348]: osdmap e126: 8 total, 8 up, 8 in 2024-04-26T16:26:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T16:26:35.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:35 smithi059 ceph-mon[86348]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T16:26:36.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:36 smithi059 ceph-mon[81151]: pgmap v180: 161 pgs: 24 active+undersized+degraded, 42 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:36.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:36 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:36.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:36 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:36.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:36 smithi059 ceph-mon[86348]: pgmap v180: 161 pgs: 24 active+undersized+degraded, 42 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:36.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:36 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:36.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:36 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:36.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:36 smithi064 ceph-mon[65271]: pgmap v180: 161 pgs: 24 active+undersized+degraded, 42 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 78/627 objects degraded (12.440%) 2024-04-26T16:26:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:36 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:36 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:37.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:37 smithi059 ceph-mon[81151]: Health check update: Degraded data redundancy: 63/627 objects degraded (10.048%), 20 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:37 smithi059 ceph-mon[86348]: Health check update: Degraded data redundancy: 63/627 objects degraded (10.048%), 20 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:37.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:37 smithi064 ceph-mon[65271]: Health check update: Degraded data redundancy: 63/627 objects degraded (10.048%), 20 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:38.327 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:38 smithi059 ceph-mon[81151]: pgmap v182: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:26:38.327 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:38 smithi059 ceph-mon[86348]: pgmap v182: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:26:38.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:38 smithi064 ceph-mon[65271]: pgmap v182: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:26:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:26:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:26:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:26:39.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:39.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:39 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:26:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:26:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:26:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:26:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:39 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:26:40.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[81151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 63/627 objects degraded (10.048%), 20 pgs degraded) 2024-04-26T16:26:40.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:40.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:40.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:40.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:40.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T16:26:40.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[86348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 63/627 objects degraded (10.048%), 20 pgs degraded) 2024-04-26T16:26:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:40.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T16:26:40.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:26:40.747+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:26:40.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:26:40.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:26:40.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:40.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:26:40.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:40.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:40.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:40.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:40.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:40.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:40.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:40.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:40.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:40 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:40 smithi064 ceph-mon[65271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 63/627 objects degraded (10.048%), 20 pgs degraded) 2024-04-26T16:26:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T16:26:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:41 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:41.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:41.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:26:41.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:41.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:41.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:41.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:41 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T16:26:41.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:41 smithi064 ceph-mon[65271]: Upgrade: osd.4 is safe to restart 2024-04-26T16:26:41.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:41 smithi064 ceph-mon[65271]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:41 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:42.083 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:42.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:26:42.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:42.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:42.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:26:42.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:42.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:42.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:42.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:42.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:41 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:42.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:42.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:42.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:42.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:42.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:26:42.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:26:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:26:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:26:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:26:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:26:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:26:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:26:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:41 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:41 smithi059 ceph-mon[81151]: Upgrade: osd.4 is safe to restart 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:41 smithi059 ceph-mon[81151]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:41 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:41 smithi059 ceph-mon[86348]: Upgrade: osd.4 is safe to restart 2024-04-26T16:26:42.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:41 smithi059 ceph-mon[86348]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:43.904 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:43 smithi064 ceph-mon[65271]: Upgrade: Updating osd.4 2024-04-26T16:26:43.923 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:43.923 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T16:26:43.923 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:43.923 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:43 smithi064 ceph-mon[65271]: Deploying daemon osd.4 on smithi064 2024-04-26T16:26:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:43 smithi059 ceph-mon[81151]: Upgrade: Updating osd.4 2024-04-26T16:26:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T16:26:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:44.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:43 smithi059 ceph-mon[81151]: Deploying daemon osd.4 on smithi064 2024-04-26T16:26:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:43 smithi059 ceph-mon[86348]: Upgrade: Updating osd.4 2024-04-26T16:26:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T16:26:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:26:44.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:43 smithi059 ceph-mon[86348]: Deploying daemon osd.4 on smithi064 2024-04-26T16:26:44.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:26:44] "GET /metrics HTTP/1.1" 200 38326 "" "Prometheus/2.43.0" 2024-04-26T16:26:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:44 smithi064 ceph-mon[65271]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:44 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:45.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:44 smithi059 ceph-mon[81151]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:44 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:44 smithi059 ceph-mon[86348]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:26:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:44 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:46.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:45 smithi064 systemd[1]: Stopping Ceph osd.4 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:26:46.593 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:46 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[26173]: 2024-04-26T16:26:46.275+0000 7f48496d5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:26:46.593 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:46 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[26173]: 2024-04-26T16:26:46.275+0000 7f48496d5700 -1 osd.4 127 *** Got signal Terminated *** 2024-04-26T16:26:46.593 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:46 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[26173]: 2024-04-26T16:26:46.275+0000 7f48496d5700 -1 osd.4 127 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:26:46.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:46 smithi064 ceph-mon[65271]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 105 B/s rd, 0 op/s 2024-04-26T16:26:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:46 smithi064 ceph-mon[65271]: osd.4 marked itself down and dead 2024-04-26T16:26:46.944 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:46 smithi059 ceph-mon[86348]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 105 B/s rd, 0 op/s 2024-04-26T16:26:46.944 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:46.944 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:46.944 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:46.944 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:46 smithi059 ceph-mon[86348]: osd.4 marked itself down and dead 2024-04-26T16:26:46.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:46 smithi059 ceph-mon[81151]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 105 B/s rd, 0 op/s 2024-04-26T16:26:46.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:46.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:26:46.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:46.945 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:46 smithi059 ceph-mon[81151]: osd.4 marked itself down and dead 2024-04-26T16:26:46.945 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:26:47.674 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:26:47.674 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (7m) 13s ago 29m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:26:47.674 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (6m) 4m ago 29m 80.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:26:47.674 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 13s ago 28m 62.3M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:26:47.674 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (8m) 4m ago 34m 461M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:26:47.674 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (18m) 13s ago 35m 524M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:26:47.674 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (4m) 13s ago 35m 56.0M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (5m) 4m ago 34m 40.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (4m) 13s ago 34m 44.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (8m) 13s ago 30m 17.0M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (8m) 4m ago 30m 17.2M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (2m) 13s ago 33m 52.9M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (2m) 13s ago 33m 44.0M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (76s) 13s ago 32m 41.2M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (18s) 13s ago 32m 12.4M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13cc20260a 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (32m) 4m ago 32m 55.5M 3248M 17.2.0 e1d6a67b021e 41416b2ce463 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (31m) 4m ago 31m 54.5M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:26:47.675 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (31m) 4m ago 31m 52.2M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:26:47.676 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (31m) 4m ago 31m 53.6M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:26:47.676 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (7m) 4m ago 30m 66.2M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:26:47.676 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (28m) 13s ago 28m 78.4M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:26:47.676 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (28m) 4m ago 28m 77.0M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:26:47.859 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:47 smithi064 ceph-mon[65271]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:26:47.859 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:47 smithi064 ceph-mon[65271]: osdmap e128: 8 total, 7 up, 8 in 2024-04-26T16:26:47.860 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:47 smithi064 podman[75410]: 2024-04-26 16:26:47.547739867 +0000 UTC m=+1.387138872 container died 41416b2ce463fdaca417fb621a259ba7cf80727fd5410d0a2ce3ef9a90ec82b7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, name=centos-stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., distribution-scope=public) 2024-04-26T16:26:48.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:47 smithi059 ceph-mon[81151]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:26:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:47 smithi059 ceph-mon[81151]: osdmap e128: 8 total, 7 up, 8 in 2024-04-26T16:26:48.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:47 smithi059 ceph-mon[86348]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:26:48.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:47 smithi059 ceph-mon[86348]: osdmap e128: 8 total, 7 up, 8 in 2024-04-26T16:26:48.103 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 4 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:26:48.104 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:26:48.105 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-26T16:26:48.105 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 9 2024-04-26T16:26:48.105 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:26:48.105 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:26:48.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:47 smithi064 podman[75410]: 2024-04-26 16:26:47.999157029 +0000 UTC m=+1.838556108 container cleanup 41416b2ce463fdaca417fb621a259ba7cf80727fd5410d0a2ce3ef9a90ec82b7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4, RELEASE=HEAD, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, distribution-scope=public, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, name=centos-stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0) 2024-04-26T16:26:48.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:47 smithi064 bash[75410]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4 2024-04-26T16:26:48.472 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:26:48.473 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:26:48.473 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:26:48.473 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-26T16:26:48.473 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:26:48.473 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "4/8 daemons upgraded", 2024-04-26T16:26:48.473 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T16:26:48.473 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:26:48.473 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:26:48.600 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:48 smithi064 podman[75439]: 2024-04-26 16:26:48.285887443 +0000 UTC m=+0.734081441 container remove 41416b2ce463fdaca417fb621a259ba7cf80727fd5410d0a2ce3ef9a90ec82b7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4, distribution-scope=public, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, version=8, GIT_BRANCH=HEAD, vcs-type=git, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True) 2024-04-26T16:26:48.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:48 smithi059 ceph-mon[86348]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-26T16:26:48.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:48 smithi059 ceph-mon[86348]: from='client.64408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:48.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:48 smithi059 ceph-mon[86348]: from='client.64414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:48.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:48 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/3577840079' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:48.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:48 smithi059 ceph-mon[86348]: osdmap e129: 8 total, 7 up, 8 in 2024-04-26T16:26:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:48 smithi059 ceph-mon[81151]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-26T16:26:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:48 smithi059 ceph-mon[81151]: from='client.64408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:48 smithi059 ceph-mon[81151]: from='client.64414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:48 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/3577840079' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:48 smithi059 ceph-mon[81151]: osdmap e129: 8 total, 7 up, 8 in 2024-04-26T16:26:48.921 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:48 smithi064 ceph-mon[65271]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-26T16:26:48.922 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:48 smithi064 ceph-mon[65271]: from='client.64408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:48.922 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:48 smithi064 ceph-mon[65271]: from='client.64414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:48.922 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:48 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/3577840079' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:26:48.922 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:48 smithi064 ceph-mon[65271]: osdmap e129: 8 total, 7 up, 8 in 2024-04-26T16:26:48.922 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:48 smithi064 podman[75533]: 2024-04-26 16:26:48.535757633 +0000 UTC m=+0.018680907 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:26:48.922 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:48 smithi064 podman[75533]: 2024-04-26 16:26:48.636789368 +0000 UTC m=+0.119712630 container create 6e8cf9673ad9156437eded000dd9c1585dd9589d46f6116047a79aa4ca0edd90 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-26T16:26:48.922 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:48 smithi064 podman[75533]: 2024-04-26 16:26:48.847575978 +0000 UTC m=+0.330499240 container init 6e8cf9673ad9156437eded000dd9c1585dd9589d46f6116047a79aa4ca0edd90 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-26T16:26:48.922 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:48 smithi064 podman[75533]: 2024-04-26 16:26:48.851025931 +0000 UTC m=+0.333949198 container start 6e8cf9673ad9156437eded000dd9c1585dd9589d46f6116047a79aa4ca0edd90 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:26:48.923 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:48 smithi064 podman[75533]: 2024-04-26 16:26:48.922325717 +0000 UTC m=+0.405248989 container attach 6e8cf9673ad9156437eded000dd9c1585dd9589d46f6116047a79aa4ca0edd90 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:26:49.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:49 smithi064 podman[75533]: 2024-04-26 16:26:49.152804045 +0000 UTC m=+0.635727307 container died 6e8cf9673ad9156437eded000dd9c1585dd9589d46f6116047a79aa4ca0edd90 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True) 2024-04-26T16:26:49.734 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:49 smithi064 ceph-mon[65271]: from='client.64420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:49.734 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:49 smithi064 ceph-mon[65271]: from='client.54498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:49.986 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:49 smithi064 podman[75559]: 2024-04-26 16:26:49.734814723 +0000 UTC m=+0.576313821 container remove 6e8cf9673ad9156437eded000dd9c1585dd9589d46f6116047a79aa4ca0edd90 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-26T16:26:49.986 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:49 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-26T16:26:49.986 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:49 smithi064 systemd[1]: Stopped Ceph osd.4 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:26:49.986 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:49 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.4.service: Consumed 18.176s CPU time. 2024-04-26T16:26:50.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:49 smithi059 ceph-mon[86348]: from='client.64420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:50.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:49 smithi059 ceph-mon[86348]: from='client.54498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:50.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:49 smithi059 ceph-mon[81151]: from='client.64420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:50.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:49 smithi059 ceph-mon[81151]: from='client.54498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:26:50.309 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:49 smithi064 systemd[1]: Starting Ceph osd.4 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:26:50.309 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 podman[75664]: 2024-04-26 16:26:50.202875803 +0000 UTC m=+0.110434016 container create f1b970a1ea525f7fed4641728f865b420f106d2cbabd7ed886385760680a64dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-26T16:26:50.309 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 podman[75664]: 2024-04-26 16:26:50.111073747 +0000 UTC m=+0.018631958 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:26:50.607 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 podman[75664]: 2024-04-26 16:26:50.43024396 +0000 UTC m=+0.337802171 container init f1b970a1ea525f7fed4641728f865b420f106d2cbabd7ed886385760680a64dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True) 2024-04-26T16:26:50.607 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 podman[75664]: 2024-04-26 16:26:50.433650467 +0000 UTC m=+0.341208678 container start f1b970a1ea525f7fed4641728f865b420f106d2cbabd7ed886385760680a64dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, ceph=True) 2024-04-26T16:26:50.607 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 podman[75664]: 2024-04-26 16:26:50.496693722 +0000 UTC m=+0.404251933 container attach f1b970a1ea525f7fed4641728f865b420f106d2cbabd7ed886385760680a64dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:26:50.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:50 smithi059 ceph-mon[86348]: pgmap v191: 161 pgs: 21 peering, 14 stale+active+clean, 126 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-26T16:26:50.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:50 smithi059 ceph-mon[86348]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-04-26T16:26:50.876 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:50 smithi059 ceph-mon[81151]: pgmap v191: 161 pgs: 21 peering, 14 stale+active+clean, 126 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-26T16:26:50.876 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:50 smithi059 ceph-mon[81151]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-04-26T16:26:50.927 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-mon[65271]: pgmap v191: 161 pgs: 21 peering, 14 stale+active+clean, 126 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-26T16:26:50.927 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-mon[65271]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-04-26T16:26:51.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate[75679]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:26:51.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate[75679]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:26:51.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 bash[75664]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:26:51.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 bash[75664]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:26:51.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 bash[75664]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T16:26:51.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 bash[75664]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 bash[75664]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 bash[75664]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 bash[75664]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 bash[75664]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 bash[75664]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate[75679]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate[75679]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate[75679]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate[75679]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate[75679]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate[75679]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate[75679]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-26T16:26:51.188 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:50 smithi064 podman[75664]: 2024-04-26 16:26:50.952890336 +0000 UTC m=+0.860448548 container died f1b970a1ea525f7fed4641728f865b420f106d2cbabd7ed886385760680a64dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-26T16:26:51.658 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:51 smithi064 podman[75817]: 2024-04-26 16:26:51.47575296 +0000 UTC m=+0.512685910 container remove f1b970a1ea525f7fed4641728f865b420f106d2cbabd7ed886385760680a64dc (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS) 2024-04-26T16:26:51.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:51 smithi064 ceph-mon[65271]: pgmap v192: 161 pgs: 2 active+undersized+degraded, 21 peering, 11 stale+active+clean, 4 active+undersized, 123 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 6/627 objects degraded (0.957%) 2024-04-26T16:26:51.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:51 smithi064 ceph-mon[65271]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:51.937 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:51 smithi064 podman[75880]: 2024-04-26 16:26:51.595578748 +0000 UTC m=+0.018856876 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:26:51.937 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:51 smithi064 podman[75880]: 2024-04-26 16:26:51.718377186 +0000 UTC m=+0.141655311 container create a170975ea549f3dbe524f248b1924b76fca3488b2161739ca2ec288e2f740ad5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-26T16:26:52.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:51 smithi059 ceph-mon[81151]: pgmap v192: 161 pgs: 2 active+undersized+degraded, 21 peering, 11 stale+active+clean, 4 active+undersized, 123 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 6/627 objects degraded (0.957%) 2024-04-26T16:26:52.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:51 smithi059 ceph-mon[81151]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:52.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:51 smithi059 ceph-mon[86348]: pgmap v192: 161 pgs: 2 active+undersized+degraded, 21 peering, 11 stale+active+clean, 4 active+undersized, 123 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 6/627 objects degraded (0.957%) 2024-04-26T16:26:52.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:51 smithi059 ceph-mon[86348]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:52.437 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:51 smithi064 podman[75880]: 2024-04-26 16:26:51.987695122 +0000 UTC m=+0.410973245 container init a170975ea549f3dbe524f248b1924b76fca3488b2161739ca2ec288e2f740ad5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T16:26:52.437 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:51 smithi064 podman[75880]: 2024-04-26 16:26:51.99125351 +0000 UTC m=+0.414531627 container start a170975ea549f3dbe524f248b1924b76fca3488b2161739ca2ec288e2f740ad5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:26:52.437 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:52 smithi064 ceph-osd[75898]: -- 172.21.15.64:0/1726934292 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559f27eb7680 con 0x559f27e81000 2024-04-26T16:26:52.437 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:52 smithi064 bash[75880]: a170975ea549f3dbe524f248b1924b76fca3488b2161739ca2ec288e2f740ad5 2024-04-26T16:26:52.437 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:52 smithi064 systemd[1]: Started Ceph osd.4 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:26:52.937 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:26:52.534+0000 7fd26344e7c0 -1 Falling back to public interface 2024-04-26T16:26:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:53 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:53 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:53.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:53 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:53 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:53 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:53 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:53 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:53 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:53 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:26:54.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:54 smithi064 ceph-mon[65271]: pgmap v193: 161 pgs: 20 active+undersized+degraded, 21 peering, 27 active+undersized, 93 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 88/627 objects degraded (14.035%) 2024-04-26T16:26:54.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:54 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:54.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:26:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:26:54] "GET /metrics HTTP/1.1" 200 38331 "" "Prometheus/2.43.0" 2024-04-26T16:26:54.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:54 smithi059 ceph-mon[81151]: pgmap v193: 161 pgs: 20 active+undersized+degraded, 21 peering, 27 active+undersized, 93 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 88/627 objects degraded (14.035%) 2024-04-26T16:26:54.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:54 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:54 smithi059 ceph-mon[86348]: pgmap v193: 161 pgs: 20 active+undersized+degraded, 21 peering, 27 active+undersized, 93 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 88/627 objects degraded (14.035%) 2024-04-26T16:26:54.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:54 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:26:55.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:55 smithi059 ceph-mon[81151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-04-26T16:26:55.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:55 smithi059 ceph-mon[86348]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-04-26T16:26:55.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:55 smithi064 ceph-mon[65271]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-04-26T16:26:55.687 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:26:55.256+0000 7fd26344e7c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-26T16:26:55.687 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:55 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:26:55.298+0000 7fd26344e7c0 -1 osd.4 127 log_to_monitors true 2024-04-26T16:26:56.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:56 smithi059 ceph-mon[81151]: pgmap v194: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-26T16:26:56.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:56 smithi059 ceph-mon[81151]: from='osd.4 [v2:172.21.15.64:6800/3875462913,v1:172.21.15.64:6801/3875462913]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T16:26:56.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:56 smithi059 ceph-mon[81151]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T16:26:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:56 smithi059 ceph-mon[86348]: pgmap v194: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-26T16:26:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:56 smithi059 ceph-mon[86348]: from='osd.4 [v2:172.21.15.64:6800/3875462913,v1:172.21.15.64:6801/3875462913]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T16:26:56.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:56 smithi059 ceph-mon[86348]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T16:26:56.661 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:56 smithi064 ceph-mon[65271]: pgmap v194: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-26T16:26:56.661 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:56 smithi064 ceph-mon[65271]: from='osd.4 [v2:172.21.15.64:6800/3875462913,v1:172.21.15.64:6801/3875462913]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T16:26:56.661 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:56 smithi064 ceph-mon[65271]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T16:26:56.936 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:26:56 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:26:56.710+0000 7fd25089e700 -1 osd.4 127 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T16:26:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[81151]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T16:26:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[81151]: osdmap e130: 8 total, 7 up, 8 in 2024-04-26T16:26:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[81151]: from='osd.4 [v2:172.21.15.64:6800/3875462913,v1:172.21.15.64:6801/3875462913]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:26:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[81151]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:26:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[81151]: from='osd.4 ' entity='osd.4' 2024-04-26T16:26:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[81151]: Health check update: Degraded data redundancy: 107/627 objects degraded (17.065%), 26 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[86348]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T16:26:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[86348]: osdmap e130: 8 total, 7 up, 8 in 2024-04-26T16:26:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[86348]: from='osd.4 [v2:172.21.15.64:6800/3875462913,v1:172.21.15.64:6801/3875462913]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:26:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[86348]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:26:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[86348]: from='osd.4 ' entity='osd.4' 2024-04-26T16:26:57.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:57 smithi059 ceph-mon[86348]: Health check update: Degraded data redundancy: 107/627 objects degraded (17.065%), 26 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:57 smithi064 ceph-mon[65271]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T16:26:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:57 smithi064 ceph-mon[65271]: osdmap e130: 8 total, 7 up, 8 in 2024-04-26T16:26:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:57 smithi064 ceph-mon[65271]: from='osd.4 [v2:172.21.15.64:6800/3875462913,v1:172.21.15.64:6801/3875462913]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:26:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:57 smithi064 ceph-mon[65271]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:26:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:57 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:57 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:57 smithi064 ceph-mon[65271]: from='osd.4 ' entity='osd.4' 2024-04-26T16:26:57.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:57 smithi064 ceph-mon[65271]: Health check update: Degraded data redundancy: 107/627 objects degraded (17.065%), 26 pgs degraded (PG_DEGRADED) 2024-04-26T16:26:58.471 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:58 smithi064 ceph-mon[65271]: pgmap v196: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 184 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-26T16:26:58.471 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:58 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:58.471 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:58 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:58.471 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:58 smithi064 ceph-mon[65271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:26:58.471 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:58 smithi064 ceph-mon[65271]: osd.4 [v2:172.21.15.64:6800/3875462913,v1:172.21.15.64:6801/3875462913] boot 2024-04-26T16:26:58.471 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:58 smithi064 ceph-mon[65271]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T16:26:58.472 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:58 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:26:58.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[81151]: pgmap v196: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 184 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-26T16:26:58.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:58.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:58.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[81151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:26:58.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[81151]: osd.4 [v2:172.21.15.64:6800/3875462913,v1:172.21.15.64:6801/3875462913] boot 2024-04-26T16:26:58.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[81151]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T16:26:58.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:26:58.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[86348]: pgmap v196: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 184 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-26T16:26:58.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:58.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:26:58.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[86348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:26:58.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[86348]: osd.4 [v2:172.21.15.64:6800/3875462913,v1:172.21.15.64:6801/3875462913] boot 2024-04-26T16:26:58.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[86348]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T16:26:58.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:58 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T16:27:00.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:59 smithi059 ceph-mon[81151]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T16:27:00.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:26:59 smithi059 ceph-mon[81151]: pgmap v199: 161 pgs: 12 peering, 23 active+undersized+degraded, 33 active+undersized, 93 active+clean; 585 KiB data, 184 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:00.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:59 smithi059 ceph-mon[86348]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T16:27:00.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:26:59 smithi059 ceph-mon[86348]: pgmap v199: 161 pgs: 12 peering, 23 active+undersized+degraded, 33 active+undersized, 93 active+clean; 585 KiB data, 184 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:00.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:59 smithi064 ceph-mon[65271]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T16:27:00.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:26:59 smithi064 ceph-mon[65271]: pgmap v199: 161 pgs: 12 peering, 23 active+undersized+degraded, 33 active+undersized, 93 active+clean; 585 KiB data, 184 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:02.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: pgmap v200: 161 pgs: 12 peering, 23 active+undersized+degraded, 30 active+undersized, 96 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:02.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:02.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:27:02.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: pgmap v201: 161 pgs: 12 peering, 23 active+undersized+degraded, 30 active+undersized, 96 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:02.048 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: pgmap v202: 161 pgs: 12 peering, 23 active+undersized+degraded, 30 active+undersized, 96 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: pgmap v200: 161 pgs: 12 peering, 23 active+undersized+degraded, 30 active+undersized, 96 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.049 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:02.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:27:02.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: pgmap v201: 161 pgs: 12 peering, 23 active+undersized+degraded, 30 active+undersized, 96 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:02.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: pgmap v202: 161 pgs: 12 peering, 23 active+undersized+degraded, 30 active+undersized, 96 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:02.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:27:02.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:27:02.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:02.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:27:02.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: pgmap v200: 161 pgs: 12 peering, 23 active+undersized+degraded, 30 active+undersized, 96 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: pgmap v201: 161 pgs: 12 peering, 23 active+undersized+degraded, 30 active+undersized, 96 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: pgmap v202: 161 pgs: 12 peering, 23 active+undersized+degraded, 30 active+undersized, 96 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 94/627 objects degraded (14.992%) 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:27:02.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:02.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:27:02.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:02.188 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[86348]: Health check update: Degraded data redundancy: 94/627 objects degraded (14.992%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:03.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[81151]: Health check update: Degraded data redundancy: 94/627 objects degraded (14.992%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:03.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:03.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:03.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:03.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:03.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:03.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:27:02.899+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:03.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:27:03.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:03.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:03.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:03.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:02 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:27:02.911+0000 7fe2d86fd700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-26T16:27:03.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:02 smithi064 ceph-mon[65271]: Health check update: Degraded data redundancy: 94/627 objects degraded (14.992%), 23 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:03.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:02 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:04.332 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:27:04] "GET /metrics HTTP/1.1" 200 38196 "" "Prometheus/2.43.0" 2024-04-26T16:27:04.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:03 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:04.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:04.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:04.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:04.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:04.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:04.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:03 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:04.335 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:27:04.336 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:27:04.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:27:04.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:27:04.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:27:04.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:27:04.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:27:04.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:27:04.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:27:04.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:27:04.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:27:04.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:27:04.338 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:27:04.338 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:27:04.338 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:27:04.338 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:27:04.338 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:04.338 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:04.338 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:04.338 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:04.338 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:04.339 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:27:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:03 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:03 smithi059 ceph-mon[86348]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:03 smithi059 ceph-mon[86348]: pgmap v203: 161 pgs: 12 peering, 149 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-04-26T16:27:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:03 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:03 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:03 smithi059 ceph-mon[81151]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-26T16:27:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:03 smithi059 ceph-mon[81151]: pgmap v203: 161 pgs: 12 peering, 149 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-04-26T16:27:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:03 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:04.346 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:03 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:04.347 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:27:04.348 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:04.349 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:04.350 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:04.351 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:04.351 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:04.351 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:04.351 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:04.351 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:04.351 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:03 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:04.351 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:03 smithi064 ceph-mon[65271]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-26T16:27:04.351 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:03 smithi064 ceph-mon[65271]: pgmap v203: 161 pgs: 12 peering, 149 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-04-26T16:27:04.351 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:03 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:05.288 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:04 smithi064 ceph-mon[65271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 94/627 objects degraded (14.992%), 23 pgs degraded) 2024-04-26T16:27:05.299 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:04 smithi059 ceph-mon[81151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 94/627 objects degraded (14.992%), 23 pgs degraded) 2024-04-26T16:27:05.299 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:04 smithi059 ceph-mon[86348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 94/627 objects degraded (14.992%), 23 pgs degraded) 2024-04-26T16:27:06.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:05 smithi059 ceph-mon[81151]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-04-26T16:27:06.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:05 smithi059 ceph-mon[86348]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-04-26T16:27:06.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:05 smithi064 ceph-mon[65271]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 822 B/s rd, 0 op/s 2024-04-26T16:27:08.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:07 smithi059 ceph-mon[86348]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-26T16:27:08.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:07 smithi059 ceph-mon[81151]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-26T16:27:08.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:07 smithi064 ceph-mon[65271]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-26T16:27:10.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:09 smithi059 ceph-mon[81151]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:10.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:09 smithi059 ceph-mon[86348]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:10.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:09 smithi064 ceph-mon[65271]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:12 smithi059 ceph-mon[81151]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:27:12.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:12 smithi059 ceph-mon[86348]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:27:12.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:12.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:12 smithi064 ceph-mon[65271]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:27:12.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:14 smithi059 ceph-mon[81151]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:14.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:14 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:14.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:27:14] "GET /metrics HTTP/1.1" 200 38196 "" "Prometheus/2.43.0" 2024-04-26T16:27:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:14 smithi059 ceph-mon[86348]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:14.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:14 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:14.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:14 smithi064 ceph-mon[65271]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:14 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:16 smithi059 ceph-mon[81151]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:16 smithi059 ceph-mon[86348]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:16.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:16.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:16 smithi064 ceph-mon[65271]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:18.536 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:18 smithi064 ceph-mon[65271]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:18.536 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:18 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:18.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:18 smithi059 ceph-mon[81151]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:18.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:18 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:18 smithi059 ceph-mon[86348]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:18.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:18 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:18.856 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:27:19.549 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:27:19.549 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (7m) 45s ago 30m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:27:19.549 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (7m) 23s ago 29m 80.5M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:27:19.549 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 45s ago 29m 62.3M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:27:19.549 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (9m) 23s ago 34m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:27:19.549 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (19m) 45s ago 36m 524M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (5m) 45s ago 36m 56.0M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (6m) 23s ago 34m 50.8M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (5m) 45s ago 34m 44.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (8m) 45s ago 30m 17.0M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (8m) 23s ago 30m 17.1M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (3m) 45s ago 34m 52.9M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (2m) 45s ago 33m 44.0M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (108s) 45s ago 33m 41.2M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (50s) 45s ago 33m 12.4M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13cc20260a 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (27s) 23s ago 32m 12.1M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd a170975ea549 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (32m) 23s ago 32m 57.8M 3248M 17.2.0 e1d6a67b021e 21ffd2ef8611 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (31m) 23s ago 31m 55.7M 3248M 17.2.0 e1d6a67b021e dabf25ce78ba 2024-04-26T16:27:19.550 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (31m) 23s ago 31m 59.1M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:27:19.551 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (8m) 23s ago 30m 69.0M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:27:19.551 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (29m) 45s ago 29m 78.4M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:27:19.551 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (29m) 23s ago 29m 78.3M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:27:19.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:19 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:19.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:19 smithi059 ceph-mon[81151]: Upgrade: osd.5 is safe to restart 2024-04-26T16:27:19.551 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:19 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:19.551 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:19 smithi059 ceph-mon[86348]: Upgrade: osd.5 is safe to restart 2024-04-26T16:27:19.584 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:19 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-26T16:27:19.584 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:19 smithi064 ceph-mon[65271]: Upgrade: osd.5 is safe to restart 2024-04-26T16:27:19.976 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:27:19.976 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:27:19.976 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:27:19.976 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:27:19.976 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 5 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-26T16:27:19.977 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 10 2024-04-26T16:27:19.978 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:27:19.978 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:27:20.329 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:27:20.329 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:27:20.329 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:27:20.329 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-26T16:27:20.329 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:27:20.329 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "5/8 daemons upgraded", 2024-04-26T16:27:20.329 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T16:27:20.330 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:27:20.330 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:27:20.330 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[81151]: from='client.64441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:20.330 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[81151]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:20.330 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[81151]: from='client.64447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:20.330 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[81151]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:20.330 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:20.330 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T16:27:20.330 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:20.330 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/2888801622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:20.331 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[86348]: from='client.64441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:20.331 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[86348]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:20.331 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[86348]: from='client.64447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:20.331 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[86348]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:20.331 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:20.331 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T16:27:20.331 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:20.331 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:20 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/2888801622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:20.478 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:20 smithi064 ceph-mon[65271]: from='client.64441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:20.478 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:20 smithi064 ceph-mon[65271]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:20.478 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:20 smithi064 ceph-mon[65271]: from='client.64447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:20.478 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:20 smithi064 ceph-mon[65271]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:20.479 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:20 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:20.479 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:20 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T16:27:20.479 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:20 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:20.479 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:20 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2888801622' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:21.423 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:21 smithi064 ceph-mon[65271]: Upgrade: Updating osd.5 2024-04-26T16:27:21.423 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:21 smithi064 ceph-mon[65271]: Deploying daemon osd.5 on smithi064 2024-04-26T16:27:21.423 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:21 smithi064 ceph-mon[65271]: from='client.54531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:21.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:21 smithi059 ceph-mon[86348]: Upgrade: Updating osd.5 2024-04-26T16:27:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:21 smithi059 ceph-mon[86348]: Deploying daemon osd.5 on smithi064 2024-04-26T16:27:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:21 smithi059 ceph-mon[86348]: from='client.54531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:21.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:21 smithi059 ceph-mon[81151]: Upgrade: Updating osd.5 2024-04-26T16:27:21.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:21 smithi059 ceph-mon[81151]: Deploying daemon osd.5 on smithi064 2024-04-26T16:27:21.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:21 smithi059 ceph-mon[81151]: from='client.54531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:22 smithi059 ceph-mon[86348]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:22.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:22 smithi059 ceph-mon[81151]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:22.615 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:22 smithi064 ceph-mon[65271]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:22.937 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:22 smithi064 systemd[1]: Stopping Ceph osd.5 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:27:23.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:23 smithi064 ceph-mon[65271]: osd.5 marked itself down and dead 2024-04-26T16:27:23.437 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[29051]: 2024-04-26T16:27:23.086+0000 7f2c12081700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:27:23.437 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[29051]: 2024-04-26T16:27:23.086+0000 7f2c12081700 -1 osd.5 132 *** Got signal Terminated *** 2024-04-26T16:27:23.437 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[29051]: 2024-04-26T16:27:23.086+0000 7f2c12081700 -1 osd.5 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:27:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:23 smithi059 ceph-mon[86348]: osd.5 marked itself down and dead 2024-04-26T16:27:23.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:23 smithi059 ceph-mon[81151]: osd.5 marked itself down and dead 2024-04-26T16:27:23.812 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:23 smithi064 podman[79991]: 2024-04-26 16:27:23.555421287 +0000 UTC m=+0.590468545 container died 21ffd2ef8611cb1807dff989df67b97c9f0411868c8b0d77827168ff743d2cb5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.openshift.expose-services=, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, com.redhat.component=centos-stream-container) 2024-04-26T16:27:24.087 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:23 smithi064 podman[79991]: 2024-04-26 16:27:23.903293799 +0000 UTC m=+0.938341066 container cleanup 21ffd2ef8611cb1807dff989df67b97c9f0411868c8b0d77827168ff743d2cb5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., release=754, com.redhat.component=centos-stream-container, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, distribution-scope=public) 2024-04-26T16:27:24.087 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:23 smithi064 bash[79991]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5 2024-04-26T16:27:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:24 smithi064 ceph-mon[65271]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:24 smithi064 ceph-mon[65271]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:27:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:24 smithi064 ceph-mon[65271]: osdmap e133: 8 total, 7 up, 8 in 2024-04-26T16:27:24.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:24 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:24.437 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:24 smithi064 podman[80019]: 2024-04-26 16:27:24.088126994 +0000 UTC m=+0.528271318 container remove 21ffd2ef8611cb1807dff989df67b97c9f0411868c8b0d77827168ff743d2cb5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5, distribution-scope=public, architecture=x86_64, version=8, io.buildah.version=1.19.8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, ceph=True, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8) 2024-04-26T16:27:24.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:27:24] "GET /metrics HTTP/1.1" 200 38338 "" "Prometheus/2.43.0" 2024-04-26T16:27:24.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:24 smithi059 ceph-mon[86348]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:24.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:24 smithi059 ceph-mon[86348]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:27:24.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:24 smithi059 ceph-mon[86348]: osdmap e133: 8 total, 7 up, 8 in 2024-04-26T16:27:24.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:24 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:24.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:24 smithi059 ceph-mon[81151]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:27:24.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:24 smithi059 ceph-mon[81151]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:27:24.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:24 smithi059 ceph-mon[81151]: osdmap e133: 8 total, 7 up, 8 in 2024-04-26T16:27:24.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:24 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:24.824 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:24 smithi064 podman[80115]: 2024-04-26 16:27:24.434303873 +0000 UTC m=+0.018368679 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:27:24.824 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:24 smithi064 podman[80115]: 2024-04-26 16:27:24.530944192 +0000 UTC m=+0.115008969 container create e60fd9ff4d592362942779c7e23df5a64ae83a2afdaa645de94e436ae32ff817 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:27:24.825 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:24 smithi064 podman[80115]: 2024-04-26 16:27:24.758508744 +0000 UTC m=+0.342573517 container init e60fd9ff4d592362942779c7e23df5a64ae83a2afdaa645de94e436ae32ff817 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:27:24.825 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:24 smithi064 podman[80115]: 2024-04-26 16:27:24.761744356 +0000 UTC m=+0.345809138 container start e60fd9ff4d592362942779c7e23df5a64ae83a2afdaa645de94e436ae32ff817 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-26T16:27:24.825 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:24 smithi064 podman[80115]: 2024-04-26 16:27:24.824967592 +0000 UTC m=+0.409032381 container attach e60fd9ff4d592362942779c7e23df5a64ae83a2afdaa645de94e436ae32ff817 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True) 2024-04-26T16:27:25.186 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:25 smithi064 podman[80115]: 2024-04-26 16:27:25.066370877 +0000 UTC m=+0.650435661 container died e60fd9ff4d592362942779c7e23df5a64ae83a2afdaa645de94e436ae32ff817 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:27:25.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:25 smithi059 ceph-mon[86348]: osdmap e134: 8 total, 7 up, 8 in 2024-04-26T16:27:25.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:25 smithi059 ceph-mon[81151]: osdmap e134: 8 total, 7 up, 8 in 2024-04-26T16:27:25.602 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:25 smithi064 ceph-mon[65271]: osdmap e134: 8 total, 7 up, 8 in 2024-04-26T16:27:25.937 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:25 smithi064 podman[80141]: 2024-04-26 16:27:25.603676606 +0000 UTC m=+0.531696490 container remove e60fd9ff4d592362942779c7e23df5a64ae83a2afdaa645de94e436ae32ff817 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T16:27:25.937 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:25 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-26T16:27:25.937 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:25 smithi064 systemd[1]: Stopped Ceph osd.5 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:27:25.937 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:25 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.5.service: Consumed 16.820s CPU time. 2024-04-26T16:27:25.937 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:25 smithi064 systemd[1]: Starting Ceph osd.5 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:27:26.334 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:26 smithi064 ceph-mon[65271]: pgmap v216: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:26.334 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 podman[80252]: 2024-04-26 16:27:25.954131941 +0000 UTC m=+0.022013067 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:27:26.334 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 podman[80252]: 2024-04-26 16:27:26.096780601 +0000 UTC m=+0.164661719 container create 33fd843120c22773e1bbd8f4194162f4cf1553125cd96061c8d36506b6a76297 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T16:27:26.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:26 smithi059 ceph-mon[86348]: pgmap v216: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:26.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:26 smithi059 ceph-mon[81151]: pgmap v216: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:27:26.686 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 podman[80252]: 2024-04-26 16:27:26.334616394 +0000 UTC m=+0.402497513 container init 33fd843120c22773e1bbd8f4194162f4cf1553125cd96061c8d36506b6a76297 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS) 2024-04-26T16:27:26.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 podman[80252]: 2024-04-26 16:27:26.338243801 +0000 UTC m=+0.406124923 container start 33fd843120c22773e1bbd8f4194162f4cf1553125cd96061c8d36506b6a76297 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T16:27:26.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 podman[80252]: 2024-04-26 16:27:26.442886183 +0000 UTC m=+0.510767310 container attach 33fd843120c22773e1bbd8f4194162f4cf1553125cd96061c8d36506b6a76297 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-26T16:27:27.154 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate[80267]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:27:27.155 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate[80267]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:27:27.155 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 bash[80252]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:27:27.155 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 bash[80252]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:27:27.155 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 bash[80252]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-26T16:27:27.155 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 bash[80252]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T16:27:27.155 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 bash[80252]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T16:27:27.155 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 bash[80252]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T16:27:27.155 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 bash[80252]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-26T16:27:27.155 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 bash[80252]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-26T16:27:27.156 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 bash[80252]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-26T16:27:27.156 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate[80267]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-26T16:27:27.156 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate[80267]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T16:27:27.156 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate[80267]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T16:27:27.156 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate[80267]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T16:27:27.156 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate[80267]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-26T16:27:27.156 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate[80267]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-26T16:27:27.156 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate[80267]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-26T16:27:27.156 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:26 smithi064 podman[80252]: 2024-04-26 16:27:26.858666524 +0000 UTC m=+0.926547654 container died 33fd843120c22773e1bbd8f4194162f4cf1553125cd96061c8d36506b6a76297 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:27:27.421 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:27 smithi064 ceph-mon[65271]: Health check failed: Degraded data redundancy: 23/627 objects degraded (3.668%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:27 smithi059 ceph-mon[86348]: Health check failed: Degraded data redundancy: 23/627 objects degraded (3.668%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:27 smithi059 ceph-mon[81151]: Health check failed: Degraded data redundancy: 23/627 objects degraded (3.668%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:27.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:27 smithi064 podman[80404]: 2024-04-26 16:27:27.421614911 +0000 UTC m=+0.557453851 container remove 33fd843120c22773e1bbd8f4194162f4cf1553125cd96061c8d36506b6a76297 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-26T16:27:27.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:27 smithi064 podman[80468]: 2024-04-26 16:27:27.546718338 +0000 UTC m=+0.019101005 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:27:27.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:27 smithi064 podman[80468]: 2024-04-26 16:27:27.656884192 +0000 UTC m=+0.129266833 container create 833404a87b94e76731f271c0cb0b83696f09b85f70fd06255ef123c03425fdd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-26T16:27:28.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:27 smithi064 podman[80468]: 2024-04-26 16:27:27.910455996 +0000 UTC m=+0.382838646 container init 833404a87b94e76731f271c0cb0b83696f09b85f70fd06255ef123c03425fdd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:27:28.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:27 smithi064 podman[80468]: 2024-04-26 16:27:27.913452401 +0000 UTC m=+0.385835043 container start 833404a87b94e76731f271c0cb0b83696f09b85f70fd06255ef123c03425fdd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T16:27:28.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:27 smithi064 ceph-osd[80486]: -- 172.21.15.64:0/1798705457 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a261335680 con 0x55a2612ff400 2024-04-26T16:27:28.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:28 smithi064 bash[80468]: 833404a87b94e76731f271c0cb0b83696f09b85f70fd06255ef123c03425fdd6 2024-04-26T16:27:28.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:28 smithi064 systemd[1]: Started Ceph osd.5 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:27:28.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:28 smithi059 ceph-mon[86348]: pgmap v217: 161 pgs: 6 active+undersized+degraded, 15 stale+active+clean, 14 active+undersized, 126 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 23/627 objects degraded (3.668%) 2024-04-26T16:27:28.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:28 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:28.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:28 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:28.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:28 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:28.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:28 smithi059 ceph-mon[81151]: pgmap v217: 161 pgs: 6 active+undersized+degraded, 15 stale+active+clean, 14 active+undersized, 126 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 23/627 objects degraded (3.668%) 2024-04-26T16:27:28.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:28 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:28.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:28 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:28.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:28 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:28.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:28 smithi064 ceph-mon[65271]: pgmap v217: 161 pgs: 6 active+undersized+degraded, 15 stale+active+clean, 14 active+undersized, 126 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 23/627 objects degraded (3.668%) 2024-04-26T16:27:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:28 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:28 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:28.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:28 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:28.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:27:28.455+0000 7f5b6595c7c0 -1 Falling back to public interface 2024-04-26T16:27:30.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:30 smithi059 ceph-mon[81151]: pgmap v218: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-26T16:27:30.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:30 smithi059 ceph-mon[86348]: pgmap v218: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-26T16:27:30.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:30 smithi064 ceph-mon[65271]: pgmap v218: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-26T16:27:31.437 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:27:31.180+0000 7f5b6595c7c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-26T16:27:31.437 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:27:31.208+0000 7f5b6595c7c0 -1 osd.5 132 log_to_monitors true 2024-04-26T16:27:32.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:32 smithi064 ceph-mon[65271]: pgmap v219: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-26T16:27:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:32 smithi064 ceph-mon[65271]: from='osd.5 [v2:172.21.15.64:6808/2321885595,v1:172.21.15.64:6809/2321885595]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T16:27:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:32 smithi064 ceph-mon[65271]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T16:27:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:32 smithi059 ceph-mon[81151]: pgmap v219: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-26T16:27:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:32 smithi059 ceph-mon[81151]: from='osd.5 [v2:172.21.15.64:6808/2321885595,v1:172.21.15.64:6809/2321885595]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T16:27:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:32 smithi059 ceph-mon[81151]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T16:27:32.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:32 smithi059 ceph-mon[86348]: pgmap v219: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-26T16:27:32.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:32.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:32.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:32 smithi059 ceph-mon[86348]: from='osd.5 [v2:172.21.15.64:6808/2321885595,v1:172.21.15.64:6809/2321885595]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T16:27:32.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:32 smithi059 ceph-mon[86348]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T16:27:32.936 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:27:32 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:27:32.600+0000 7f5b52dac700 -1 osd.5 132 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T16:27:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:33 smithi064 ceph-mon[65271]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T16:27:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:33 smithi064 ceph-mon[65271]: osdmap e135: 8 total, 7 up, 8 in 2024-04-26T16:27:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:33 smithi064 ceph-mon[65271]: from='osd.5 [v2:172.21.15.64:6808/2321885595,v1:172.21.15.64:6809/2321885595]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:33 smithi064 ceph-mon[65271]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:33 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:33 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:33.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:33 smithi064 ceph-mon[65271]: from='osd.5 ' entity='osd.5' 2024-04-26T16:27:33.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[86348]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T16:27:33.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[86348]: osdmap e135: 8 total, 7 up, 8 in 2024-04-26T16:27:33.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[86348]: from='osd.5 [v2:172.21.15.64:6808/2321885595,v1:172.21.15.64:6809/2321885595]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:33.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[86348]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:33.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:33.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:33.484 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[86348]: from='osd.5 ' entity='osd.5' 2024-04-26T16:27:33.485 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[81151]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T16:27:33.485 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[81151]: osdmap e135: 8 total, 7 up, 8 in 2024-04-26T16:27:33.485 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[81151]: from='osd.5 [v2:172.21.15.64:6808/2321885595,v1:172.21.15.64:6809/2321885595]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:33.485 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[81151]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:33.485 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:33.485 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:33.485 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:33 smithi059 ceph-mon[81151]: from='osd.5 ' entity='osd.5' 2024-04-26T16:27:34.479 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:27:34] "GET /metrics HTTP/1.1" 200 38341 "" "Prometheus/2.43.0" 2024-04-26T16:27:34.616 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:34 smithi064 ceph-mon[65271]: pgmap v221: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-26T16:27:34.616 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:34 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:34.616 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:34 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:34.616 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:34 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:34.616 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:34 smithi064 ceph-mon[65271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:27:34.616 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:34 smithi064 ceph-mon[65271]: osd.5 [v2:172.21.15.64:6808/2321885595,v1:172.21.15.64:6809/2321885595] boot 2024-04-26T16:27:34.616 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:34 smithi064 ceph-mon[65271]: osdmap e136: 8 total, 8 up, 8 in 2024-04-26T16:27:34.616 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:34 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:27:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[81151]: pgmap v221: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-26T16:27:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[81151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:27:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[81151]: osd.5 [v2:172.21.15.64:6808/2321885595,v1:172.21.15.64:6809/2321885595] boot 2024-04-26T16:27:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[81151]: osdmap e136: 8 total, 8 up, 8 in 2024-04-26T16:27:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:27:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[86348]: pgmap v221: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-26T16:27:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[86348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:27:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[86348]: osd.5 [v2:172.21.15.64:6808/2321885595,v1:172.21.15.64:6809/2321885595] boot 2024-04-26T16:27:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[86348]: osdmap e136: 8 total, 8 up, 8 in 2024-04-26T16:27:34.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:34 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T16:27:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:35 smithi064 ceph-mon[65271]: osdmap e137: 8 total, 8 up, 8 in 2024-04-26T16:27:36.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:35 smithi059 ceph-mon[81151]: osdmap e137: 8 total, 8 up, 8 in 2024-04-26T16:27:36.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:35 smithi059 ceph-mon[86348]: osdmap e137: 8 total, 8 up, 8 in 2024-04-26T16:27:36.840 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:36 smithi064 ceph-mon[65271]: pgmap v224: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 73/627 objects degraded (11.643%) 2024-04-26T16:27:37.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:36 smithi059 ceph-mon[81151]: pgmap v224: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 73/627 objects degraded (11.643%) 2024-04-26T16:27:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:36 smithi059 ceph-mon[86348]: pgmap v224: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 73/627 objects degraded (11.643%) 2024-04-26T16:27:37.963 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: pgmap v225: 161 pgs: 17 active+undersized+degraded, 27 active+undersized, 117 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: pgmap v226: 161 pgs: 17 active+undersized+degraded, 27 active+undersized, 117 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:27:37.964 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[86348]: Health check update: Degraded data redundancy: 73/627 objects degraded (11.643%), 21 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: pgmap v225: 161 pgs: 17 active+undersized+degraded, 27 active+undersized, 117 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: pgmap v226: 161 pgs: 17 active+undersized+degraded, 27 active+undersized, 117 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:27:37.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:37 smithi059 ceph-mon[81151]: Health check update: Degraded data redundancy: 73/627 objects degraded (11.643%), 21 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:38.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: pgmap v225: 161 pgs: 17 active+undersized+degraded, 27 active+undersized, 117 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: pgmap v226: 161 pgs: 17 active+undersized+degraded, 27 active+undersized, 117 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:27:38.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:37 smithi064 ceph-mon[65271]: Health check update: Degraded data redundancy: 73/627 objects degraded (11.643%), 21 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:39.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-mon[81151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 63/627 objects degraded (10.048%), 17 pgs degraded) 2024-04-26T16:27:39.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-mon[86348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 63/627 objects degraded (10.048%), 17 pgs degraded) 2024-04-26T16:27:39.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:27:39.039+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:39.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:27:39.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:39.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:39.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:39.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:39.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:39.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:39.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:27:39.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:27:39.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:27:39.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:27:39.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:39.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:39.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:39.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:39.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:39.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:39.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:39.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:39 smithi064 ceph-mon[65271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 63/627 objects degraded (10.048%), 17 pgs degraded) 2024-04-26T16:27:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:40 smithi064 ceph-mon[65271]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-04-26T16:27:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:40 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.187 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.188 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:27:40.189 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:27:40.190 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T16:27:40.191 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:40 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T16:27:40.192 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:40 smithi064 ceph-mon[65271]: Upgrade: osd.6 is safe to restart 2024-04-26T16:27:40.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[81151]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-04-26T16:27:40.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:40.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:40.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:40.333 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:40.334 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:27:40.335 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.336 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:40.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:40.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:40.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:40.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:27:40.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:40.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:40.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:40.337 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:40.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[86348]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:40.338 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:27:40.339 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:27:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:27:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:27:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[81151]: Upgrade: osd.6 is safe to restart 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T16:27:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-26T16:27:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:40 smithi059 ceph-mon[86348]: Upgrade: osd.6 is safe to restart 2024-04-26T16:27:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:41 smithi059 ceph-mon[86348]: Upgrade: Updating osd.6 2024-04-26T16:27:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:41 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:41 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T16:27:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:41 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:41 smithi059 ceph-mon[86348]: Deploying daemon osd.6 on smithi064 2024-04-26T16:27:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:41 smithi059 ceph-mon[81151]: Upgrade: Updating osd.6 2024-04-26T16:27:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:41 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:41 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T16:27:41.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:41 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:41.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:41 smithi059 ceph-mon[81151]: Deploying daemon osd.6 on smithi064 2024-04-26T16:27:41.870 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:41 smithi064 ceph-mon[65271]: Upgrade: Updating osd.6 2024-04-26T16:27:41.871 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:41 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:41.871 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:41 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T16:27:41.871 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:41 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:41.871 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:41 smithi064 ceph-mon[65271]: Deploying daemon osd.6 on smithi064 2024-04-26T16:27:42.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:42 smithi059 ceph-mon[86348]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 977 B/s rd, 0 op/s 2024-04-26T16:27:42.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:42 smithi059 ceph-mon[81151]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 977 B/s rd, 0 op/s 2024-04-26T16:27:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:42 smithi064 ceph-mon[65271]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 977 B/s rd, 0 op/s 2024-04-26T16:27:43.933 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:43 smithi064 systemd[1]: Stopping Ceph osd.6 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:27:44.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[31927]: 2024-04-26T16:27:43.933+0000 7fc597504700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:27:44.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[31927]: 2024-04-26T16:27:43.933+0000 7fc597504700 -1 osd.6 137 *** Got signal Terminated *** 2024-04-26T16:27:44.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:43 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[31927]: 2024-04-26T16:27:43.933+0000 7fc597504700 -1 osd.6 137 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:27:44.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:27:44] "GET /metrics HTTP/1.1" 200 38341 "" "Prometheus/2.43.0" 2024-04-26T16:27:44.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:44 smithi064 ceph-mon[65271]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-04-26T16:27:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:44 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:44 smithi064 ceph-mon[65271]: osd.6 marked itself down and dead 2024-04-26T16:27:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:44 smithi059 ceph-mon[81151]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-04-26T16:27:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:44 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:44 smithi059 ceph-mon[81151]: osd.6 marked itself down and dead 2024-04-26T16:27:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:44 smithi059 ceph-mon[86348]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-04-26T16:27:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:44 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:44 smithi059 ceph-mon[86348]: osd.6 marked itself down and dead 2024-04-26T16:27:45.291 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:44 smithi064 podman[85017]: 2024-04-26 16:27:44.9854817 +0000 UTC m=+1.146576577 container died dabf25ce78ba97bb2eedde7d96a494c4a10bbe12c49d5a108860e7fb46da8e60 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6, GIT_BRANCH=HEAD, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, ceph=True, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.component=centos-stream-container) 2024-04-26T16:27:45.585 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:45 smithi064 podman[85017]: 2024-04-26 16:27:45.291605066 +0000 UTC m=+1.452699945 container cleanup dabf25ce78ba97bb2eedde7d96a494c4a10bbe12c49d5a108860e7fb46da8e60 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, version=8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-26T16:27:45.586 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:45 smithi064 bash[85017]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6 2024-04-26T16:27:45.586 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:45 smithi064 podman[85036]: 2024-04-26 16:27:45.49283096 +0000 UTC m=+0.503869813 container remove dabf25ce78ba97bb2eedde7d96a494c4a10bbe12c49d5a108860e7fb46da8e60 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6, release=754, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, version=8, io.openshift.tags=base centos centos-stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., ceph=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-26T16:27:45.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:45 smithi064 ceph-mon[65271]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:27:45.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:45 smithi064 ceph-mon[65271]: osdmap e138: 8 total, 7 up, 8 in 2024-04-26T16:27:46.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:45 smithi059 ceph-mon[81151]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:27:46.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:45 smithi059 ceph-mon[81151]: osdmap e138: 8 total, 7 up, 8 in 2024-04-26T16:27:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:45 smithi059 ceph-mon[86348]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:27:46.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:45 smithi059 ceph-mon[86348]: osdmap e138: 8 total, 7 up, 8 in 2024-04-26T16:27:46.270 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:45 smithi064 podman[85129]: 2024-04-26 16:27:45.943542097 +0000 UTC m=+0.100222054 container create 997f5af2eec3b74bcfb5b1f9d29981f261e607b40d81bbaa45ae7e476dabc8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-26T16:27:46.270 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:45 smithi064 podman[85129]: 2024-04-26 16:27:45.861488863 +0000 UTC m=+0.018168837 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:27:46.270 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:46 smithi064 podman[85129]: 2024-04-26 16:27:46.179755027 +0000 UTC m=+0.336434999 container init 997f5af2eec3b74bcfb5b1f9d29981f261e607b40d81bbaa45ae7e476dabc8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-26T16:27:46.271 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:46 smithi064 podman[85129]: 2024-04-26 16:27:46.18332768 +0000 UTC m=+0.340007636 container start 997f5af2eec3b74bcfb5b1f9d29981f261e607b40d81bbaa45ae7e476dabc8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T16:27:46.271 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:46 smithi064 podman[85129]: 2024-04-26 16:27:46.271033442 +0000 UTC m=+0.427713407 container attach 997f5af2eec3b74bcfb5b1f9d29981f261e607b40d81bbaa45ae7e476dabc8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD) 2024-04-26T16:27:46.602 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:46 smithi064 podman[85129]: 2024-04-26 16:27:46.48443994 +0000 UTC m=+0.641119895 container died 997f5af2eec3b74bcfb5b1f9d29981f261e607b40d81bbaa45ae7e476dabc8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T16:27:46.852 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:46 smithi064 ceph-mon[65271]: pgmap v231: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-26T16:27:46.852 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:46 smithi064 ceph-mon[65271]: osdmap e139: 8 total, 7 up, 8 in 2024-04-26T16:27:46.853 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:46.853 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:46.853 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:46 smithi059 ceph-mon[81151]: pgmap v231: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-26T16:27:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:46 smithi059 ceph-mon[81151]: osdmap e139: 8 total, 7 up, 8 in 2024-04-26T16:27:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:47.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:46 smithi059 ceph-mon[86348]: pgmap v231: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-26T16:27:47.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:46 smithi059 ceph-mon[86348]: osdmap e139: 8 total, 7 up, 8 in 2024-04-26T16:27:47.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:47.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:27:47.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:47.103 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:47 smithi064 podman[85155]: 2024-04-26 16:27:47.083491527 +0000 UTC m=+0.593884459 container remove 997f5af2eec3b74bcfb5b1f9d29981f261e607b40d81bbaa45ae7e476dabc8a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T16:27:47.436 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:47 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-26T16:27:47.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:47 smithi064 systemd[1]: Stopped Ceph osd.6 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:27:47.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:47 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.6.service: Consumed 15.721s CPU time. 2024-04-26T16:27:47.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:47 smithi064 systemd[1]: Starting Ceph osd.6 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:27:47.836 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:47 smithi064 ceph-mon[65271]: Health check failed: Degraded data redundancy: 26/627 objects degraded (4.147%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:47.836 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:47 smithi064 podman[85258]: 2024-04-26 16:27:47.433066789 +0000 UTC m=+0.018361720 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:27:47.836 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:47 smithi064 podman[85258]: 2024-04-26 16:27:47.534668397 +0000 UTC m=+0.119963342 container create de1e96ded3416ec05886d18da92f8391da2c78225fe02bbbd274f57d7fe7a47f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:27:47.836 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:47 smithi064 podman[85258]: 2024-04-26 16:27:47.754397766 +0000 UTC m=+0.339692670 container init de1e96ded3416ec05886d18da92f8391da2c78225fe02bbbd274f57d7fe7a47f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:27:47.836 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:47 smithi064 podman[85258]: 2024-04-26 16:27:47.757832545 +0000 UTC m=+0.343127450 container start de1e96ded3416ec05886d18da92f8391da2c78225fe02bbbd274f57d7fe7a47f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:27:47.836 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:47 smithi064 podman[85258]: 2024-04-26 16:27:47.836762466 +0000 UTC m=+0.422057384 container attach de1e96ded3416ec05886d18da92f8391da2c78225fe02bbbd274f57d7fe7a47f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:27:48.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:47 smithi059 ceph-mon[81151]: Health check failed: Degraded data redundancy: 26/627 objects degraded (4.147%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:48.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:47 smithi059 ceph-mon[86348]: Health check failed: Degraded data redundancy: 26/627 objects degraded (4.147%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:48.565 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate[85273]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:27:48.565 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate[85273]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:27:48.565 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 bash[85258]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:27:48.565 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 bash[85258]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:27:48.565 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 bash[85258]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-26T16:27:48.565 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 bash[85258]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T16:27:48.565 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 bash[85258]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T16:27:48.565 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 bash[85258]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T16:27:48.565 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 bash[85258]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-26T16:27:48.566 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 bash[85258]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-26T16:27:48.566 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 bash[85258]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-26T16:27:48.566 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate[85273]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-26T16:27:48.566 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate[85273]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T16:27:48.566 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate[85273]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T16:27:48.566 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate[85273]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T16:27:48.566 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate[85273]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-26T16:27:48.566 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate[85273]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-26T16:27:48.566 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate[85273]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-26T16:27:48.566 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 podman[85258]: 2024-04-26 16:27:48.279010289 +0000 UTC m=+0.864305209 container died de1e96ded3416ec05886d18da92f8391da2c78225fe02bbbd274f57d7fe7a47f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:27:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:48 smithi064 ceph-mon[65271]: pgmap v233: 161 pgs: 6 active+undersized+degraded, 5 stale+active+clean, 12 active+undersized, 138 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 26/627 objects degraded (4.147%) 2024-04-26T16:27:48.937 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:48 smithi064 podman[85411]: 2024-04-26 16:27:48.807690818 +0000 UTC m=+0.523096460 container remove de1e96ded3416ec05886d18da92f8391da2c78225fe02bbbd274f57d7fe7a47f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-26T16:27:49.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:48 smithi059 ceph-mon[86348]: pgmap v233: 161 pgs: 6 active+undersized+degraded, 5 stale+active+clean, 12 active+undersized, 138 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 26/627 objects degraded (4.147%) 2024-04-26T16:27:49.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:48 smithi059 ceph-mon[81151]: pgmap v233: 161 pgs: 6 active+undersized+degraded, 5 stale+active+clean, 12 active+undersized, 138 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 26/627 objects degraded (4.147%) 2024-04-26T16:27:49.285 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:49 smithi064 podman[85476]: 2024-04-26 16:27:48.928123454 +0000 UTC m=+0.018737708 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:27:49.285 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:49 smithi064 podman[85476]: 2024-04-26 16:27:49.041954721 +0000 UTC m=+0.132568971 container create ab006709251fb65dfbf4933d2d2d0e96ab8bc30ffb47c3735b4f2241b9b4206e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True) 2024-04-26T16:27:49.687 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:49 smithi064 podman[85476]: 2024-04-26 16:27:49.286215247 +0000 UTC m=+0.376829498 container init ab006709251fb65dfbf4933d2d2d0e96ab8bc30ffb47c3735b4f2241b9b4206e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-26T16:27:49.687 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:49 smithi064 podman[85476]: 2024-04-26 16:27:49.289846577 +0000 UTC m=+0.380460836 container start ab006709251fb65dfbf4933d2d2d0e96ab8bc30ffb47c3735b4f2241b9b4206e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T16:27:49.687 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:49 smithi064 ceph-osd[85495]: -- 172.21.15.64:0/753209865 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561d4ce3f680 con 0x561d4ce09400 2024-04-26T16:27:49.687 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:49 smithi064 bash[85476]: ab006709251fb65dfbf4933d2d2d0e96ab8bc30ffb47c3735b4f2241b9b4206e 2024-04-26T16:27:49.687 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:49 smithi064 systemd[1]: Started Ceph osd.6 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:27:50.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:49 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:27:49.834+0000 7f213f4e27c0 -1 Falling back to public interface 2024-04-26T16:27:50.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:50 smithi064 ceph-mon[65271]: pgmap v234: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:50.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:50.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:50.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:50 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:50.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:50 smithi059 ceph-mon[81151]: pgmap v234: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:50.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:50.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:50.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:50 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:50.709 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:50 smithi059 ceph-mon[86348]: pgmap v234: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:50.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:50.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:50.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:50 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:27:50.710 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:27:51.402 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:27:51.402 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (8m) 77s ago 30m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:27:51.402 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (8m) 19s ago 30m 81.3M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:27:51.402 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 77s ago 29m 62.3M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:27:51.402 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (9m) 19s ago 35m 465M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:27:51.402 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (19m) 77s ago 36m 524M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (5m) 77s ago 36m 56.0M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (6m) 19s ago 35m 52.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (5m) 77s ago 35m 44.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (9m) 77s ago 31m 17.0M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (9m) 19s ago 31m 17.1M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (3m) 77s ago 34m 52.9M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (3m) 77s ago 34m 44.0M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (2m) 77s ago 33m 41.2M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (82s) 77s ago 33m 12.4M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13cc20260a 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (59s) 19s ago 33m 51.1M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd a170975ea549 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (23s) 19s ago 32m 11.6M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 833404a87b94 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 starting - - - 3248M 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (32m) 19s ago 32m 59.7M 3248M 17.2.0 e1d6a67b021e de997a7c3891 2024-04-26T16:27:51.403 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (8m) 19s ago 31m 69.0M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:27:51.404 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (29m) 77s ago 29m 78.4M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:27:51.404 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (29m) 19s ago 29m 78.5M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:27:51.828 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:27:51.828 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:27:51.828 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:27:51.828 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:27:51.828 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:27:51.828 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:27:51.828 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:27:51.828 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:27:51.828 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-26T16:27:51.828 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 6 2024-04-26T16:27:51.829 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:27:51.829 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:27:51.829 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:27:51.829 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:27:51.829 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:27:51.829 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-26T16:27:51.829 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 11 2024-04-26T16:27:51.829 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:27:51.829 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:27:52.182 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:27:52.183 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:27:52.183 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:27:52.183 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-26T16:27:52.183 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:27:52.183 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "6/8 daemons upgraded", 2024-04-26T16:27:52.183 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Currently upgrading osd daemons", 2024-04-26T16:27:52.183 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:27:52.183 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:27:52.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:52 smithi059 ceph-mon[86348]: from='client.64477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:52.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:52 smithi059 ceph-mon[86348]: pgmap v235: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:52.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:52 smithi059 ceph-mon[86348]: from='client.54546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:52.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:52 smithi059 ceph-mon[86348]: from='client.64489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:52.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:52 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/671560162' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:52.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:52 smithi059 ceph-mon[81151]: from='client.64477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:52.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:52 smithi059 ceph-mon[81151]: pgmap v235: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:52.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:52 smithi059 ceph-mon[81151]: from='client.54546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:52.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:52 smithi059 ceph-mon[81151]: from='client.64489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:52.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:52 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/671560162' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:52.682 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:52 smithi064 ceph-mon[65271]: from='client.64477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:52.683 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:52 smithi064 ceph-mon[65271]: pgmap v235: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:52.683 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:52 smithi064 ceph-mon[65271]: from='client.54546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:52.683 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:52 smithi064 ceph-mon[65271]: from='client.64489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:52.683 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:52 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/671560162' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:27:52.683 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:27:52.555+0000 7f213f4e27c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-26T16:27:52.683 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:27:52.588+0000 7f213f4e27c0 -1 osd.6 137 log_to_monitors true 2024-04-26T16:27:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:53 smithi059 ceph-mon[86348]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:53 smithi059 ceph-mon[86348]: from='osd.6 [v2:172.21.15.64:6816/4226671228,v1:172.21.15.64:6817/4226671228]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T16:27:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:53 smithi059 ceph-mon[86348]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T16:27:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:53 smithi059 ceph-mon[81151]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:53 smithi059 ceph-mon[81151]: from='osd.6 [v2:172.21.15.64:6816/4226671228,v1:172.21.15.64:6817/4226671228]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T16:27:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:53 smithi059 ceph-mon[81151]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T16:27:53.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:53 smithi064 ceph-mon[65271]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:27:53.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:53 smithi064 ceph-mon[65271]: from='osd.6 [v2:172.21.15.64:6816/4226671228,v1:172.21.15.64:6817/4226671228]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T16:27:53.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:53 smithi064 ceph-mon[65271]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T16:27:54.186 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:27:53 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:27:53.897+0000 7f212c932700 -1 osd.6 137 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T16:27:54.430 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:27:54] "GET /metrics HTTP/1.1" 200 38347 "" "Prometheus/2.43.0" 2024-04-26T16:27:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:54 smithi064 ceph-mon[65271]: pgmap v236: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:54 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:54 smithi064 ceph-mon[65271]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T16:27:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:54 smithi064 ceph-mon[65271]: from='osd.6 [v2:172.21.15.64:6816/4226671228,v1:172.21.15.64:6817/4226671228]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:54 smithi064 ceph-mon[65271]: osdmap e140: 8 total, 7 up, 8 in 2024-04-26T16:27:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:54 smithi064 ceph-mon[65271]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:54 smithi064 ceph-mon[65271]: from='osd.6 ' entity='osd.6' 2024-04-26T16:27:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:54 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:54.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:54 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[86348]: pgmap v236: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[86348]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T16:27:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[86348]: from='osd.6 [v2:172.21.15.64:6816/4226671228,v1:172.21.15.64:6817/4226671228]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[86348]: osdmap e140: 8 total, 7 up, 8 in 2024-04-26T16:27:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[86348]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[86348]: from='osd.6 ' entity='osd.6' 2024-04-26T16:27:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[81151]: pgmap v236: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:27:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[81151]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T16:27:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[81151]: from='osd.6 [v2:172.21.15.64:6816/4226671228,v1:172.21.15.64:6817/4226671228]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[81151]: osdmap e140: 8 total, 7 up, 8 in 2024-04-26T16:27:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[81151]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:27:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[81151]: from='osd.6 ' entity='osd.6' 2024-04-26T16:27:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:54.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:54 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:55.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:55 smithi064 ceph-mon[65271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:27:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:55 smithi064 ceph-mon[65271]: osd.6 [v2:172.21.15.64:6816/4226671228,v1:172.21.15.64:6817/4226671228] boot 2024-04-26T16:27:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:55 smithi064 ceph-mon[65271]: osdmap e141: 8 total, 8 up, 8 in 2024-04-26T16:27:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:55 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:27:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:55 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:55 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[86348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:27:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[86348]: osd.6 [v2:172.21.15.64:6816/4226671228,v1:172.21.15.64:6817/4226671228] boot 2024-04-26T16:27:55.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[86348]: osdmap e141: 8 total, 8 up, 8 in 2024-04-26T16:27:55.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:27:55.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:55.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:55.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[81151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:27:55.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[81151]: osd.6 [v2:172.21.15.64:6816/4226671228,v1:172.21.15.64:6817/4226671228] boot 2024-04-26T16:27:55.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[81151]: osdmap e141: 8 total, 8 up, 8 in 2024-04-26T16:27:55.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T16:27:55.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:55.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:55 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:56.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:56 smithi064 ceph-mon[65271]: pgmap v239: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:56.677 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:56 smithi064 ceph-mon[65271]: osdmap e142: 8 total, 8 up, 8 in 2024-04-26T16:27:56.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:56 smithi059 ceph-mon[81151]: pgmap v239: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:56.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:56 smithi059 ceph-mon[81151]: osdmap e142: 8 total, 8 up, 8 in 2024-04-26T16:27:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:56 smithi059 ceph-mon[86348]: pgmap v239: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-04-26T16:27:56.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:56 smithi059 ceph-mon[86348]: osdmap e142: 8 total, 8 up, 8 in 2024-04-26T16:27:57.796 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:57 smithi064 ceph-mon[65271]: Health check update: Degraded data redundancy: 77/627 objects degraded (12.281%), 20 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:57.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:57 smithi059 ceph-mon[81151]: Health check update: Degraded data redundancy: 77/627 objects degraded (12.281%), 20 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:57.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:57 smithi059 ceph-mon[86348]: Health check update: Degraded data redundancy: 77/627 objects degraded (12.281%), 20 pgs degraded (PG_DEGRADED) 2024-04-26T16:27:58.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:58 smithi059 ceph-mon[81151]: pgmap v241: 161 pgs: 17 active+undersized+degraded, 26 active+undersized, 118 active+clean; 585 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:27:58.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:58 smithi059 ceph-mon[86348]: pgmap v241: 161 pgs: 17 active+undersized+degraded, 26 active+undersized, 118 active+clean; 585 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:27:58.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:58 smithi064 ceph-mon[65271]: pgmap v241: 161 pgs: 17 active+undersized+degraded, 26 active+undersized, 118 active+clean; 585 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 63/627 objects degraded (10.048%) 2024-04-26T16:27:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:27:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[81151]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-26T16:27:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[81151]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 594 B/s rd, 0 op/s 2024-04-26T16:27:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[86348]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[86348]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 594 B/s rd, 0 op/s 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:27:59.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:27:59 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:59.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:27:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:27:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:59 smithi064 ceph-mon[65271]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-26T16:27:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:59 smithi064 ceph-mon[65271]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 594 B/s rd, 0 op/s 2024-04-26T16:27:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:27:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:27:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:27:59 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:00.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:00.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:00.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-mon[81151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 63/627 objects degraded (10.048%), 17 pgs degraded) 2024-04-26T16:28:00.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:00.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:00.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-mon[86348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 63/627 objects degraded (10.048%), 17 pgs degraded) 2024-04-26T16:28:00.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:28:00.605+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:00.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:28:00.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:00.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:00.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:00.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:28:00.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:00.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:00.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:00.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:00 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:00.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:00 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:00.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:00 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:00.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:00 smithi064 ceph-mon[65271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 63/627 objects degraded (10.048%), 17 pgs degraded) 2024-04-26T16:28:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-04-26T16:28:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:01.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:01.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:01.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:01.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: Upgrade: osd.7 is safe to restart 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:01.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:02.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-04-26T16:28:02.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:02.083 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:02.083 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:02.083 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:02.083 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:02.084 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:02.085 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:02.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:02.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:02.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:02.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:02.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:02.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:02.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:02.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:02.087 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:02.088 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:02.089 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: Upgrade: osd.7 is safe to restart 2024-04-26T16:28:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T16:28:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-26T16:28:02.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: Upgrade: osd.7 is safe to restart 2024-04-26T16:28:02.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:02.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[86348]: Upgrade: Updating osd.7 2024-04-26T16:28:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T16:28:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[86348]: Deploying daemon osd.7 on smithi064 2024-04-26T16:28:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[86348]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-26T16:28:03.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[81151]: Upgrade: Updating osd.7 2024-04-26T16:28:03.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:03.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T16:28:03.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:03.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[81151]: Deploying daemon osd.7 on smithi064 2024-04-26T16:28:03.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:03 smithi059 ceph-mon[81151]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-26T16:28:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:03 smithi064 ceph-mon[65271]: Upgrade: Updating osd.7 2024-04-26T16:28:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:03 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:03 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T16:28:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:03 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:03 smithi064 ceph-mon[65271]: Deploying daemon osd.7 on smithi064 2024-04-26T16:28:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:03 smithi064 ceph-mon[65271]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-26T16:28:04.494 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:28:04] "GET /metrics HTTP/1.1" 200 38360 "" "Prometheus/2.43.0" 2024-04-26T16:28:04.670 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:04 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:04.778 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:04 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:04.778 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:04 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:05.777 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:05 smithi064 ceph-mon[65271]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 0 B/s wr, 115 op/s 2024-04-26T16:28:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:05 smithi059 ceph-mon[81151]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 0 B/s wr, 115 op/s 2024-04-26T16:28:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:05 smithi059 ceph-mon[86348]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 0 B/s wr, 115 op/s 2024-04-26T16:28:06.029 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:05 smithi064 systemd[1]: Stopping Ceph osd.7 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:28:06.436 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[34915]: 2024-04-26T16:28:06.029+0000 7f7d15264700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:28:06.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[34915]: 2024-04-26T16:28:06.029+0000 7f7d15264700 -1 osd.7 142 *** Got signal Terminated *** 2024-04-26T16:28:06.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[34915]: 2024-04-26T16:28:06.029+0000 7f7d15264700 -1 osd.7 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:28:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:06 smithi059 ceph-mon[81151]: osd.7 marked itself down and dead 2024-04-26T16:28:06.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:06 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:06.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:06 smithi059 ceph-mon[86348]: osd.7 marked itself down and dead 2024-04-26T16:28:06.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:06 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:06.909 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:06 smithi064 ceph-mon[65271]: osd.7 marked itself down and dead 2024-04-26T16:28:06.910 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:06 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:07.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:06 smithi064 podman[89740]: 2024-04-26 16:28:06.910352738 +0000 UTC m=+0.966753620 container died de997a7c38912bb1e3638f2c93d59c09f532cec903a00888418deaa4b2360322 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, ceph=True, io.buildah.version=1.19.8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, release=754, version=8, architecture=x86_64, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container) 2024-04-26T16:28:07.501 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:07 smithi064 podman[89740]: 2024-04-26 16:28:07.20409026 +0000 UTC m=+1.260491154 container cleanup de997a7c38912bb1e3638f2c93d59c09f532cec903a00888418deaa4b2360322 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, ceph=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, architecture=x86_64, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, version=8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, name=centos-stream) 2024-04-26T16:28:07.501 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:07 smithi064 bash[89740]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7 2024-04-26T16:28:07.501 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:07 smithi064 podman[89757]: 2024-04-26 16:28:07.407436176 +0000 UTC m=+0.495097490 container remove de997a7c38912bb1e3638f2c93d59c09f532cec903a00888418deaa4b2360322 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7, vcs-type=git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_CLEAN=True, version=8, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8) 2024-04-26T16:28:07.790 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:07 smithi064 podman[89847]: 2024-04-26 16:28:07.65318072 +0000 UTC m=+0.018177999 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:28:07.791 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:07 smithi064 ceph-mon[65271]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:28:07.791 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:07 smithi064 ceph-mon[65271]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-26T16:28:07.791 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:07 smithi064 ceph-mon[65271]: osdmap e143: 8 total, 7 up, 8 in 2024-04-26T16:28:07.791 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:07 smithi064 ceph-mon[65271]: pgmap v248: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 0 B/s wr, 150 op/s 2024-04-26T16:28:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:07 smithi059 ceph-mon[81151]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:28:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:07 smithi059 ceph-mon[81151]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-26T16:28:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:07 smithi059 ceph-mon[81151]: osdmap e143: 8 total, 7 up, 8 in 2024-04-26T16:28:07.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:07 smithi059 ceph-mon[81151]: pgmap v248: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 0 B/s wr, 150 op/s 2024-04-26T16:28:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:07 smithi059 ceph-mon[86348]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T16:28:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:07 smithi059 ceph-mon[86348]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-26T16:28:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:07 smithi059 ceph-mon[86348]: osdmap e143: 8 total, 7 up, 8 in 2024-04-26T16:28:07.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:07 smithi059 ceph-mon[86348]: pgmap v248: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 0 B/s wr, 150 op/s 2024-04-26T16:28:08.085 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:07 smithi064 podman[89847]: 2024-04-26 16:28:07.791769432 +0000 UTC m=+0.156766689 container create 44963c0a4eb05b8b8afa01634dbf8ece2379310a63b405b3e0713bc9043daf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_CLEAN=True) 2024-04-26T16:28:08.085 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:08 smithi064 podman[89847]: 2024-04-26 16:28:08.002487801 +0000 UTC m=+0.367485057 container init 44963c0a4eb05b8b8afa01634dbf8ece2379310a63b405b3e0713bc9043daf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T16:28:08.085 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:08 smithi064 podman[89847]: 2024-04-26 16:28:08.006031371 +0000 UTC m=+0.371028636 container start 44963c0a4eb05b8b8afa01634dbf8ece2379310a63b405b3e0713bc9043daf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T16:28:08.085 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:08 smithi064 podman[89847]: 2024-04-26 16:28:08.08588709 +0000 UTC m=+0.450884362 container attach 44963c0a4eb05b8b8afa01634dbf8ece2379310a63b405b3e0713bc9043daf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:28:08.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:08 smithi064 podman[89847]: 2024-04-26 16:28:08.307297454 +0000 UTC m=+0.672294713 container died 44963c0a4eb05b8b8afa01634dbf8ece2379310a63b405b3e0713bc9043daf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:28:08.830 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:08 smithi064 ceph-mon[65271]: osdmap e144: 8 total, 7 up, 8 in 2024-04-26T16:28:08.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:08 smithi059 ceph-mon[81151]: osdmap e144: 8 total, 7 up, 8 in 2024-04-26T16:28:08.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:08 smithi059 ceph-mon[86348]: osdmap e144: 8 total, 7 up, 8 in 2024-04-26T16:28:09.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:08 smithi064 podman[89872]: 2024-04-26 16:28:08.830989138 +0000 UTC m=+0.517457483 container remove 44963c0a4eb05b8b8afa01634dbf8ece2379310a63b405b3e0713bc9043daf22 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5) 2024-04-26T16:28:09.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:08 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-26T16:28:09.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:08 smithi064 systemd[1]: Stopped Ceph osd.7 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:28:09.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:08 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.7.service: Consumed 18.181s CPU time. 2024-04-26T16:28:09.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:09 smithi064 systemd[1]: Starting Ceph osd.7 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:28:09.519 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:09 smithi064 podman[89980]: 2024-04-26 16:28:09.169183137 +0000 UTC m=+0.018056468 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:28:09.519 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:09 smithi064 podman[89980]: 2024-04-26 16:28:09.282632331 +0000 UTC m=+0.131505633 container create 25c099f42546fc21b8806df4a7e4d8c6719096fd04bdae397fd0100cb105e286 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T16:28:09.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:09 smithi059 ceph-mon[81151]: pgmap v250: 161 pgs: 9 active+undersized+degraded, 9 peering, 13 stale+active+clean, 20 active+undersized, 110 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 167 op/s; 22/627 objects degraded (3.509%) 2024-04-26T16:28:09.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:09 smithi059 ceph-mon[86348]: pgmap v250: 161 pgs: 9 active+undersized+degraded, 9 peering, 13 stale+active+clean, 20 active+undersized, 110 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 167 op/s; 22/627 objects degraded (3.509%) 2024-04-26T16:28:09.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:09 smithi064 ceph-mon[65271]: pgmap v250: 161 pgs: 9 active+undersized+degraded, 9 peering, 13 stale+active+clean, 20 active+undersized, 110 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 167 op/s; 22/627 objects degraded (3.509%) 2024-04-26T16:28:09.937 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:09 smithi064 podman[89980]: 2024-04-26 16:28:09.548364843 +0000 UTC m=+0.397238146 container init 25c099f42546fc21b8806df4a7e4d8c6719096fd04bdae397fd0100cb105e286 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:28:09.937 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:09 smithi064 podman[89980]: 2024-04-26 16:28:09.55193086 +0000 UTC m=+0.400804171 container start 25c099f42546fc21b8806df4a7e4d8c6719096fd04bdae397fd0100cb105e286 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux ) 2024-04-26T16:28:09.937 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:09 smithi064 podman[89980]: 2024-04-26 16:28:09.626646893 +0000 UTC m=+0.475520213 container attach 25c099f42546fc21b8806df4a7e4d8c6719096fd04bdae397fd0100cb105e286 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:28:10.355 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate[89994]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:28:10.355 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate[89994]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:28:10.355 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 bash[89980]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:28:10.355 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 bash[89980]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-26T16:28:10.355 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 bash[89980]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-26T16:28:10.355 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 bash[89980]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T16:28:10.355 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 bash[89980]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T16:28:10.355 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 bash[89980]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T16:28:10.355 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 bash[89980]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-26T16:28:10.355 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 bash[89980]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-26T16:28:10.356 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 bash[89980]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-26T16:28:10.356 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate[89994]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-26T16:28:10.356 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate[89994]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T16:28:10.356 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate[89994]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T16:28:10.356 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate[89994]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T16:28:10.356 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate[89994]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-26T16:28:10.356 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate[89994]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-26T16:28:10.356 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate[89994]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-26T16:28:10.356 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 podman[89980]: 2024-04-26 16:28:10.068293669 +0000 UTC m=+0.917166982 container died 25c099f42546fc21b8806df4a7e4d8c6719096fd04bdae397fd0100cb105e286 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD) 2024-04-26T16:28:10.630 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-mon[65271]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T16:28:10.631 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:10 smithi064 ceph-mon[65271]: Health check failed: Degraded data redundancy: 22/627 objects degraded (3.509%), 9 pgs degraded (PG_DEGRADED) 2024-04-26T16:28:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:10 smithi059 ceph-mon[81151]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T16:28:10.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:10 smithi059 ceph-mon[81151]: Health check failed: Degraded data redundancy: 22/627 objects degraded (3.509%), 9 pgs degraded (PG_DEGRADED) 2024-04-26T16:28:10.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:10 smithi059 ceph-mon[86348]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T16:28:10.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:10 smithi059 ceph-mon[86348]: Health check failed: Degraded data redundancy: 22/627 objects degraded (3.509%), 9 pgs degraded (PG_DEGRADED) 2024-04-26T16:28:10.936 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 podman[90132]: 2024-04-26 16:28:10.630369869 +0000 UTC m=+0.557001746 container remove 25c099f42546fc21b8806df4a7e4d8c6719096fd04bdae397fd0100cb105e286 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424) 2024-04-26T16:28:10.937 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 podman[90198]: 2024-04-26 16:28:10.753913249 +0000 UTC m=+0.019302429 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:28:10.937 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:10 smithi064 podman[90198]: 2024-04-26 16:28:10.856263261 +0000 UTC m=+0.121652424 container create 5757a13fa26cf0fc5c6290f1cb326f7d82ec5e69d3b2771df8a5412e2424eb45 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True) 2024-04-26T16:28:11.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:11 smithi064 podman[90198]: 2024-04-26 16:28:11.109402834 +0000 UTC m=+0.374792006 container init 5757a13fa26cf0fc5c6290f1cb326f7d82ec5e69d3b2771df8a5412e2424eb45 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:28:11.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:11 smithi064 podman[90198]: 2024-04-26 16:28:11.112939886 +0000 UTC m=+0.378329051 container start 5757a13fa26cf0fc5c6290f1cb326f7d82ec5e69d3b2771df8a5412e2424eb45 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T16:28:11.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:11 smithi064 ceph-osd[90218]: -- 172.21.15.64:0/546442557 <== mon.2 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a984a43680 con 0x55a984a0d400 2024-04-26T16:28:11.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:11 smithi064 bash[90198]: 5757a13fa26cf0fc5c6290f1cb326f7d82ec5e69d3b2771df8a5412e2424eb45 2024-04-26T16:28:11.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:11 smithi064 systemd[1]: Started Ceph osd.7 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:28:11.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:11 smithi059 ceph-mon[81151]: pgmap v251: 161 pgs: 10 active+undersized+degraded, 9 peering, 10 stale+active+clean, 26 active+undersized, 106 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 0 B/s wr, 107 op/s; 23/627 objects degraded (3.668%) 2024-04-26T16:28:11.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:11.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:11.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:11 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:11.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:11 smithi059 ceph-mon[86348]: pgmap v251: 161 pgs: 10 active+undersized+degraded, 9 peering, 10 stale+active+clean, 26 active+undersized, 106 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 0 B/s wr, 107 op/s; 23/627 objects degraded (3.668%) 2024-04-26T16:28:11.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:11.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:11.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:11 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:11.936 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:28:11.655+0000 7fe0db5a77c0 -1 Falling back to public interface 2024-04-26T16:28:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:11 smithi064 ceph-mon[65271]: pgmap v251: 161 pgs: 10 active+undersized+degraded, 9 peering, 10 stale+active+clean, 26 active+undersized, 106 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 0 B/s wr, 107 op/s; 23/627 objects degraded (3.668%) 2024-04-26T16:28:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:11 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:13.039 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:12 smithi064 ceph-mon[65271]: pgmap v252: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 49 op/s; 80/627 objects degraded (12.759%) 2024-04-26T16:28:13.039 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:12 smithi064 ceph-mon[65271]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T16:28:13.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:12 smithi059 ceph-mon[81151]: pgmap v252: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 49 op/s; 80/627 objects degraded (12.759%) 2024-04-26T16:28:13.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:12 smithi059 ceph-mon[81151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T16:28:13.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:12 smithi059 ceph-mon[86348]: pgmap v252: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 49 op/s; 80/627 objects degraded (12.759%) 2024-04-26T16:28:13.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:12 smithi059 ceph-mon[86348]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T16:28:14.035 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:13 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:14.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:13 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:14.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:13 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:14.582 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:28:14] "GET /metrics HTTP/1.1" 200 38360 "" "Prometheus/2.43.0" 2024-04-26T16:28:14.662 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:28:14.368+0000 7fe0db5a77c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-26T16:28:14.662 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:28:14.398+0000 7fe0db5a77c0 -1 osd.7 142 log_to_monitors true 2024-04-26T16:28:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:14 smithi064 ceph-mon[65271]: from='osd.7 [v2:172.21.15.64:6824/1786388927,v1:172.21.15.64:6825/1786388927]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T16:28:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:14 smithi064 ceph-mon[65271]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T16:28:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:14 smithi064 ceph-mon[65271]: pgmap v253: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 0 B/s wr, 25 op/s; 80/627 objects degraded (12.759%) 2024-04-26T16:28:15.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:14 smithi059 ceph-mon[81151]: from='osd.7 [v2:172.21.15.64:6824/1786388927,v1:172.21.15.64:6825/1786388927]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T16:28:15.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:14 smithi059 ceph-mon[81151]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T16:28:15.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:14 smithi059 ceph-mon[81151]: pgmap v253: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 0 B/s wr, 25 op/s; 80/627 objects degraded (12.759%) 2024-04-26T16:28:15.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:14 smithi059 ceph-mon[86348]: from='osd.7 [v2:172.21.15.64:6824/1786388927,v1:172.21.15.64:6825/1786388927]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T16:28:15.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:14 smithi059 ceph-mon[86348]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T16:28:15.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:14 smithi059 ceph-mon[86348]: pgmap v253: 161 pgs: 25 active+undersized+degraded, 44 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 0 B/s wr, 25 op/s; 80/627 objects degraded (12.759%) 2024-04-26T16:28:15.451 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:28:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:28:15.189+0000 7fe0c89f7700 -1 osd.7 142 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T16:28:16.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:15 smithi059 ceph-mon[81151]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T16:28:16.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:15 smithi059 ceph-mon[81151]: from='osd.7 [v2:172.21.15.64:6824/1786388927,v1:172.21.15.64:6825/1786388927]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:28:16.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:15 smithi059 ceph-mon[81151]: osdmap e145: 8 total, 7 up, 8 in 2024-04-26T16:28:16.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:15 smithi059 ceph-mon[81151]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:28:16.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:15 smithi059 ceph-mon[81151]: from='osd.7 ' entity='osd.7' 2024-04-26T16:28:16.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:15 smithi059 ceph-mon[86348]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T16:28:16.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:15 smithi059 ceph-mon[86348]: from='osd.7 [v2:172.21.15.64:6824/1786388927,v1:172.21.15.64:6825/1786388927]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:28:16.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:15 smithi059 ceph-mon[86348]: osdmap e145: 8 total, 7 up, 8 in 2024-04-26T16:28:16.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:15 smithi059 ceph-mon[86348]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:28:16.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:15 smithi059 ceph-mon[86348]: from='osd.7 ' entity='osd.7' 2024-04-26T16:28:16.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:15 smithi064 ceph-mon[65271]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T16:28:16.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:15 smithi064 ceph-mon[65271]: from='osd.7 [v2:172.21.15.64:6824/1786388927,v1:172.21.15.64:6825/1786388927]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:28:16.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:15 smithi064 ceph-mon[65271]: osdmap e145: 8 total, 7 up, 8 in 2024-04-26T16:28:16.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:15 smithi064 ceph-mon[65271]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-04-26T16:28:16.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:15 smithi064 ceph-mon[65271]: from='osd.7 ' entity='osd.7' 2024-04-26T16:28:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[81151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:28:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[81151]: osd.7 [v2:172.21.15.64:6824/1786388927,v1:172.21.15.64:6825/1786388927] boot 2024-04-26T16:28:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[81151]: osdmap e146: 8 total, 8 up, 8 in 2024-04-26T16:28:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:28:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[81151]: pgmap v256: 161 pgs: 4 peering, 22 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 248 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 0 B/s wr, 8 op/s; 68/627 objects degraded (10.845%) 2024-04-26T16:28:17.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:17.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:17.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:17.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:17.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[86348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:28:17.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[86348]: osd.7 [v2:172.21.15.64:6824/1786388927,v1:172.21.15.64:6825/1786388927] boot 2024-04-26T16:28:17.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[86348]: osdmap e146: 8 total, 8 up, 8 in 2024-04-26T16:28:17.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:28:17.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:16 smithi059 ceph-mon[86348]: pgmap v256: 161 pgs: 4 peering, 22 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 248 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 0 B/s wr, 8 op/s; 68/627 objects degraded (10.845%) 2024-04-26T16:28:17.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:17.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:17.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:17.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:17.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:16 smithi064 ceph-mon[65271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T16:28:17.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:16 smithi064 ceph-mon[65271]: osd.7 [v2:172.21.15.64:6824/1786388927,v1:172.21.15.64:6825/1786388927] boot 2024-04-26T16:28:17.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:16 smithi064 ceph-mon[65271]: osdmap e146: 8 total, 8 up, 8 in 2024-04-26T16:28:17.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T16:28:17.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:16 smithi064 ceph-mon[65271]: pgmap v256: 161 pgs: 4 peering, 22 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 248 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 0 B/s wr, 8 op/s; 68/627 objects degraded (10.845%) 2024-04-26T16:28:18.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:17 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:18.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:17 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:18.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:17 smithi064 ceph-mon[65271]: Health check update: Degraded data redundancy: 80/627 objects degraded (12.759%), 25 pgs degraded (PG_DEGRADED) 2024-04-26T16:28:18.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:17 smithi064 ceph-mon[65271]: osdmap e147: 8 total, 8 up, 8 in 2024-04-26T16:28:18.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:17 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:18.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:17 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:18.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:17 smithi059 ceph-mon[86348]: Health check update: Degraded data redundancy: 80/627 objects degraded (12.759%), 25 pgs degraded (PG_DEGRADED) 2024-04-26T16:28:18.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:17 smithi059 ceph-mon[86348]: osdmap e147: 8 total, 8 up, 8 in 2024-04-26T16:28:18.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:17 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:18.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:17 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:18.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:17 smithi059 ceph-mon[81151]: Health check update: Degraded data redundancy: 80/627 objects degraded (12.759%), 25 pgs degraded (PG_DEGRADED) 2024-04-26T16:28:18.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:17 smithi059 ceph-mon[81151]: osdmap e147: 8 total, 8 up, 8 in 2024-04-26T16:28:19.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:18 smithi064 ceph-mon[65271]: pgmap v258: 161 pgs: 10 peering, 14 active+undersized+degraded, 23 active+undersized, 114 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 47/627 objects degraded (7.496%) 2024-04-26T16:28:19.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:18 smithi059 ceph-mon[86348]: pgmap v258: 161 pgs: 10 peering, 14 active+undersized+degraded, 23 active+undersized, 114 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 47/627 objects degraded (7.496%) 2024-04-26T16:28:19.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:18 smithi059 ceph-mon[81151]: pgmap v258: 161 pgs: 10 peering, 14 active+undersized+degraded, 23 active+undersized, 114 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 47/627 objects degraded (7.496%) 2024-04-26T16:28:21.395 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:21.395 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:21.395 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:21.395 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: pgmap v259: 161 pgs: 10 peering, 3 active+undersized+degraded, 7 active+undersized, 141 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 905 B/s rd, 0 op/s; 7/627 objects degraded (1.116%) 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: pgmap v260: 161 pgs: 10 peering, 3 active+undersized+degraded, 7 active+undersized, 141 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s; 7/627 objects degraded (1.116%) 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:21.396 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:28:21.397 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: pgmap v259: 161 pgs: 10 peering, 3 active+undersized+degraded, 7 active+undersized, 141 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 905 B/s rd, 0 op/s; 7/627 objects degraded (1.116%) 2024-04-26T16:28:21.397 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: pgmap v260: 161 pgs: 10 peering, 3 active+undersized+degraded, 7 active+undersized, 141 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s; 7/627 objects degraded (1.116%) 2024-04-26T16:28:21.397 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:21.397 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:21.397 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:28:21.397 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:21.397 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:21 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:28:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: pgmap v259: 161 pgs: 10 peering, 3 active+undersized+degraded, 7 active+undersized, 141 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 905 B/s rd, 0 op/s; 7/627 objects degraded (1.116%) 2024-04-26T16:28:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: pgmap v260: 161 pgs: 10 peering, 3 active+undersized+degraded, 7 active+undersized, 141 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s; 7/627 objects degraded (1.116%) 2024-04-26T16:28:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:21.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:21.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:28:21.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:21.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:21 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:22.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-mon[86348]: Health check update: Degraded data redundancy: 7/627 objects degraded (1.116%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T16:28:22.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-mon[81151]: Health check update: Degraded data redundancy: 7/627 objects degraded (1.116%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T16:28:22.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:28:22.415+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:22.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:22.584 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:22.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:22.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:22.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:22.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:22.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:22.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:22.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:22.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:22.587 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:22.588 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:22 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:22.594 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:28:22.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:22 smithi064 ceph-mon[65271]: Health check update: Degraded data redundancy: 7/627 objects degraded (1.116%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (8m) 109s ago 31m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (8m) 7s ago 30m 79.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 109s ago 30m 62.3M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (10m) 7s ago 35m 466M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (20m) 109s ago 37m 524M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (6m) 109s ago 37m 56.0M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (7m) 7s ago 35m 52.9M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (6m) 109s ago 36m 44.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (9m) 109s ago 31m 17.0M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (9m) 7s ago 31m 16.9M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:28:23.295 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (4m) 109s ago 35m 52.9M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:28:23.296 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (3m) 109s ago 34m 44.0M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:28:23.296 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (2m) 109s ago 34m 41.2M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:28:23.296 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (114s) 109s ago 34m 12.4M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13cc20260a 2024-04-26T16:28:23.296 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (91s) 7s ago 33m 54.2M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd a170975ea549 2024-04-26T16:28:23.296 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (55s) 7s ago 33m 51.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 833404a87b94 2024-04-26T16:28:23.296 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (34s) 7s ago 32m 51.8M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd ab006709251f 2024-04-26T16:28:23.296 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (12s) 7s ago 32m 12.6M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 5757a13fa26c 2024-04-26T16:28:23.296 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (9m) 7s ago 31m 69.0M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:28:23.296 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (30m) 109s ago 30m 78.4M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:28:23.296 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (30m) 7s ago 30m 78.9M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:28:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-04-26T16:28:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:23.584 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:28:23.585 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.586 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:23.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:23.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:23.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:23.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:23.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:23.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:23.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:23.587 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:23.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-04-26T16:28:23.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:23.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:23.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:23.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:23.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:23.589 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:23.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:23.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:23.593 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all osd 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-26T16:28:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: Upgrade: Setting require_osd_release to 19 squid 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-26T16:28:23.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[86348]: from='client.64516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all osd 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-26T16:28:23.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-26T16:28:23.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-26T16:28:23.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-26T16:28:23.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-26T16:28:23.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: Upgrade: Setting require_osd_release to 19 squid 2024-04-26T16:28:23.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-26T16:28:23.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:23 smithi059 ceph-mon[81151]: from='client.64516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-04-26T16:28:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:23.688 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:23.689 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.690 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:23.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all osd 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-26T16:28:23.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: Upgrade: Setting require_osd_release to 19 squid 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-26T16:28:23.693 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:23 smithi064 ceph-mon[65271]: from='client.64516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:23.743 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:28:23.743 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:28:23.743 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:28:23.743 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:28:23.743 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:28:23.743 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 13 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:28:23.744 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:28:24.103 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:28:24.103 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": null, 2024-04-26T16:28:24.103 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": false, 2024-04-26T16:28:24.103 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "", 2024-04-26T16:28:24.103 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:28:24.103 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": null, 2024-04-26T16:28:24.103 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "", 2024-04-26T16:28:24.103 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:28:24.104 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:28:24.354 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:28:24] "GET /metrics HTTP/1.1" 200 38291 "" "Prometheus/2.43.0" 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='client.64522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/627 objects degraded (1.116%), 3 pgs degraded) 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='client.54591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: osdmap e148: 8 total, 8 up, 8 in 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mds 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all nfs 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: Upgrade: Finalizing container_image settings 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='client.64522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/627 objects degraded (1.116%), 3 pgs degraded) 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='client.54591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-26T16:28:24.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: osdmap e148: 8 total, 8 up, 8 in 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mds 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all nfs 2024-04-26T16:28:24.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: Upgrade: Finalizing container_image settings 2024-04-26T16:28:24.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:28:24.360 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:28:24.360 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.360 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:28:24.360 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:28:24.360 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:28:24.360 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:28:24.360 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-26T16:28:24.360 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:28:24.360 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:28:24.360 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.361 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:28:24.362 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: Upgrade: Complete! 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/3724011609' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.364 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: Upgrade: Complete! 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:28:24.365 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:24.366 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:24 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/3724011609' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='client.64522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:24.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/627 objects degraded (1.116%), 3 pgs degraded) 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='client.54591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: osdmap e148: 8 total, 8 up, 8 in 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mds 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all nfs 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: Upgrade: Finalizing container_image settings 2024-04-26T16:28:24.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:28:24.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.691 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: Upgrade: Complete! 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:24.692 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:24 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/3724011609' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:28:25.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:25.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:25.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[86348]: from='client.64540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:25.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:25.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:25.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:28:25.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:25.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:25.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:25.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[81151]: from='client.64540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:25.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:25.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:25.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:28:25.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:28:25.545+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:28:25.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:25.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:28:25.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:28:25.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:25.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:25.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:25.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:25.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:25.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:25.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:25.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:25.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:25 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:25 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:25 smithi064 ceph-mon[65271]: from='client.64540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:25 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:28:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:25 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:25 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:28:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:25 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:26.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:26.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:28:26.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:26.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:26.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:26.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:26.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:26.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:26.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:26.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:26.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:26.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:26.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:26.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:26.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:26.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:26.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:26.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:28:26.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:26.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:26.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:26.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:26.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:26.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:26.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:26.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:26.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[81151]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-26T16:28:26.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[81151]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-26T16:28:26.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:28:26.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:26.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:26.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:26.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:26.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:26.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.844 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[86348]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-26T16:28:26.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[86348]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-26T16:28:26.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:28:26.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:26.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:26 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:26 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:26.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:28:26.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:26.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:26.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:26 smithi064 ceph-mon[65271]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-26T16:28:26.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:26 smithi064 ceph-mon[65271]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-26T16:28:26.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:26 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:28:26.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:26 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:28:26.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:26 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:27.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:27.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:27.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:28:27.559+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:27.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:28:27.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:27.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:27.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:27.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:27.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:27.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:27.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:27.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:28:27.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:27.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:28:27.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:27.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:27.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:27.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:27.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:27.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:28:27.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:27.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:27.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:27.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:27.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:27.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:27.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:27.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:27.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:27 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:28:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:27 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:28:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:28 smithi064 ceph-mon[65271]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-26T16:28:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:28 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:28.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:28.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:28.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:28.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:29.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:28 smithi059 ceph-mon[86348]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-26T16:28:29.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:28 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:29.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:29.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:29.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:29.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:29.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:29.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:29.083 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:29.084 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:29.085 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:29.086 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:28 smithi059 ceph-mon[81151]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:28 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:29.087 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:29.088 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:28:29.089 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:28:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:28:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:28:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:28:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:28:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:28:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:28:30.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:30 smithi064 ceph-mon[65271]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 842 B/s rd, 0 op/s 2024-04-26T16:28:31.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:30 smithi059 ceph-mon[86348]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 842 B/s rd, 0 op/s 2024-04-26T16:28:31.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:30 smithi059 ceph-mon[81151]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 842 B/s rd, 0 op/s 2024-04-26T16:28:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:32 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:32.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:32 smithi064 ceph-mon[65271]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 642 B/s rd, 0 op/s 2024-04-26T16:28:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:32 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:32.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:32 smithi059 ceph-mon[81151]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 642 B/s rd, 0 op/s 2024-04-26T16:28:32.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:32.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:32.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:32 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:28:32.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:32 smithi059 ceph-mon[86348]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 642 B/s rd, 0 op/s 2024-04-26T16:28:34.505 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:28:34] "GET /metrics HTTP/1.1" 200 38450 "" "Prometheus/2.43.0" 2024-04-26T16:28:34.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:34 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:34.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:34 smithi059 ceph-mon[81151]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:28:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:34 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:34.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:34 smithi059 ceph-mon[86348]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:28:34.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:34 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:34.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:34 smithi064 ceph-mon[65271]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:28:36.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:36 smithi064 ceph-mon[65271]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:28:37.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:36 smithi059 ceph-mon[81151]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:28:37.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:36 smithi059 ceph-mon[86348]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:28:38.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:38 smithi064 ceph-mon[65271]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:39.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:38 smithi059 ceph-mon[81151]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:39.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:38 smithi059 ceph-mon[86348]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:40.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:40 smithi064 ceph-mon[65271]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:41.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:40 smithi059 ceph-mon[81151]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:41.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:40 smithi059 ceph-mon[86348]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:42.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:42 smithi064 ceph-mon[65271]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:43.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:42 smithi059 ceph-mon[81151]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:43.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:42 smithi059 ceph-mon[86348]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:44.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:28:44] "GET /metrics HTTP/1.1" 200 38450 "" "Prometheus/2.43.0" 2024-04-26T16:28:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:44 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:44 smithi064 ceph-mon[65271]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:45.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:44 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:45.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:44 smithi059 ceph-mon[86348]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:44 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:45.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:44 smithi059 ceph-mon[81151]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:46 smithi064 ceph-mon[65271]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:47.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:46 smithi059 ceph-mon[86348]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:47.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:46 smithi059 ceph-mon[81151]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:47.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:28:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:48 smithi064 ceph-mon[65271]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:49.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:48 smithi059 ceph-mon[81151]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:49.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:48 smithi059 ceph-mon[86348]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:50.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:50 smithi064 ceph-mon[65271]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:51.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:50 smithi059 ceph-mon[81151]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:51.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:50 smithi059 ceph-mon[86348]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:52.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:52 smithi064 ceph-mon[65271]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:53.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:52 smithi059 ceph-mon[81151]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:53.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:52 smithi059 ceph-mon[86348]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:54.485 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:28:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:28:54] "GET /metrics HTTP/1.1" 200 38446 "" "Prometheus/2.43.0" 2024-04-26T16:28:54.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:54 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:54.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:54 smithi059 ceph-mon[81151]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:54 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:54.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:54 smithi059 ceph-mon[86348]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:54.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:54 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:28:54.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:54 smithi064 ceph-mon[65271]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:28:55.244 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:28:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:55 smithi059 ceph-mon[81151]: from='client.54609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:55 smithi059 ceph-mon[86348]: from='client.54609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:55.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:55 smithi064 ceph-mon[65271]: from='client.54609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:56.798 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:56 smithi059 ceph-mon[81151]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:56.798 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:56 smithi059 ceph-mon[86348]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:56.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:56 smithi064 ceph-mon[65271]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (9m) 2m ago 31m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (9m) 41s ago 31m 79.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 2m ago 30m 62.3M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (10m) 41s ago 36m 466M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (20m) 2m ago 37m 524M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (7m) 2m ago 37m 56.0M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (7m) 41s ago 36m 52.9M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (6m) 2m ago 36m 44.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (10m) 2m ago 32m 17.0M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:28:57.292 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (10m) 41s ago 32m 16.9M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (4m) 2m ago 35m 52.9M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (4m) 2m ago 35m 44.0M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (3m) 2m ago 35m 41.2M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (2m) 2m ago 34m 12.4M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13cc20260a 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (2m) 41s ago 34m 54.2M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd a170975ea549 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (89s) 41s ago 33m 51.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 833404a87b94 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (68s) 41s ago 33m 51.8M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd ab006709251f 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (46s) 41s ago 33m 12.6M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 5757a13fa26c 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (9m) 41s ago 32m 69.0M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (30m) 2m ago 30m 78.4M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:28:57.293 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (30m) 41s ago 30m 78.9M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:28:58.151 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-04-26T16:28:58.788 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:58 smithi059 ceph-mon[86348]: from='client.64552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:58.788 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:28:58 smithi059 ceph-mon[86348]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:58.788 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:58 smithi059 ceph-mon[81151]: from='client.64552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:58.788 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:28:58 smithi059 ceph-mon[81151]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:28:58.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:58 smithi064 ceph-mon[65271]: from='client.64552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:28:58.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:28:58 smithi064 ceph-mon[65271]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:00.318 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:29:00.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:00 smithi064 ceph-mon[65271]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:00.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:00 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/3498297349' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:00.991 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:00 smithi059 ceph-mon[81151]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:00.991 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:00 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/3498297349' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:00.992 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:00 smithi059 ceph-mon[86348]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:00.992 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:00 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/3498297349' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:01.021 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-04-26T16:29:01.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:02.065 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:02.066 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:02.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:02 smithi064 ceph-mon[65271]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:03.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:02 smithi059 ceph-mon[81151]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:03.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:02 smithi059 ceph-mon[86348]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:03.194 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)" 2024-04-26T16:29:03.785 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade status' 2024-04-26T16:29:03.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:03 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/3377405044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:04.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:03 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/3377405044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:04.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:03 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/3377405044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:04.426 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:29:04] "GET /metrics HTTP/1.1" 200 38449 "" "Prometheus/2.43.0" 2024-04-26T16:29:04.789 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:04 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:04.789 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:04 smithi059 ceph-mon[81151]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:04.790 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:04 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:04.790 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:04 smithi059 ceph-mon[86348]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:04.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:04 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:04.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:04 smithi064 ceph-mon[65271]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:05.812 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:29:05.812 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": null, 2024-04-26T16:29:05.812 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": false, 2024-04-26T16:29:05.813 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "", 2024-04-26T16:29:05.813 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:29:05.813 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": null, 2024-04-26T16:29:05.813 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "", 2024-04-26T16:29:05.813 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:29:05.813 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:29:06.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:05 smithi059 ceph-mon[86348]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:06.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:05 smithi059 ceph-mon[81151]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:06.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:05 smithi064 ceph-mon[65271]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:06.415 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:29:06.970 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:06 smithi059 ceph-mon[81151]: from='client.64570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:06.970 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:06 smithi059 ceph-mon[86348]: from='client.64570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:07.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:06 smithi064 ceph-mon[65271]: from='client.64570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:07.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:07 smithi059 ceph-mon[86348]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:07.876 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:07 smithi059 ceph-mon[81151]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:08.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:07 smithi064 ceph-mon[65271]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:08.428 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:29:08.428 INFO:teuthology.orchestra.run.smithi059.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:29:08.428 INFO:teuthology.orchestra.run.smithi059.stdout: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:29:09.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:08 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/1805510625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:29:09.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:08 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/1805510625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:29:09.182 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-04-26T16:29:09.186 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:08 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/1805510625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:29:09.754 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:09 smithi059 ceph-mon[86348]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:09.754 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:09 smithi059 ceph-mon[81151]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:10.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:09 smithi064 ceph-mon[65271]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:12.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:12 smithi059 ceph-mon[86348]: from='client.54645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:12.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:12 smithi059 ceph-mon[86348]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:12 smithi059 ceph-mon[81151]: from='client.54645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:12.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:12 smithi059 ceph-mon[81151]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:12.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:12 smithi064 ceph-mon[65271]: from='client.54645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:12.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:12 smithi064 ceph-mon[65271]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:13.557 INFO:teuthology.orchestra.run.smithi059.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:29:14.252 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:29:14] "GET /metrics HTTP/1.1" 200 38449 "" "Prometheus/2.43.0" 2024-04-26T16:29:14.278 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-26T16:29:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[81151]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:29:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[81151]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:14.832 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[86348]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[86348]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:14.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:14.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:14.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:14 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:14 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:14 smithi064 ceph-mon[65271]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:29:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:14 smithi064 ceph-mon[65271]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:15.750 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:15 smithi059 ceph-mon[81151]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-04-26T16:29:15.751 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:15 smithi059 ceph-mon[81151]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:15.751 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:15 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:15.751 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:15 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:15.751 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:15 smithi059 ceph-mon[86348]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-04-26T16:29:15.751 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:15 smithi059 ceph-mon[86348]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:15.751 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:15 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:15.751 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:15 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:15 smithi064 ceph-mon[65271]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-04-26T16:29:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:15 smithi064 ceph-mon[65271]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:15 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:15 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:16.247 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:29:16.720 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:16.721 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:16.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:16.958 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:29:16.958 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (9m) 2m ago 32m 29.8M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:29:16.958 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (9m) 61s ago 31m 79.1M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:29:16.958 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 2m ago 31m 62.3M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:29:16.958 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (11m) 61s ago 36m 466M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:29:16.958 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (21m) 2m ago 38m 524M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (7m) 2m ago 38m 56.0M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (8m) 61s ago 36m 52.9M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (6m) 2m ago 36m 44.2M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (10m) 2m ago 32m 17.0M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (10m) 61s ago 32m 16.9M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (5m) 2m ago 36m 52.9M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (4m) 2m ago 35m 44.0M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (3m) 2m ago 35m 41.2M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (2m) 2m ago 35m 12.4M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13cc20260a 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (2m) 61s ago 34m 54.2M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd a170975ea549 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (109s) 61s ago 34m 51.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 833404a87b94 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (88s) 61s ago 33m 51.8M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd ab006709251f 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (66s) 61s ago 33m 12.6M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 5757a13fa26c 2024-04-26T16:29:16.959 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (10m) 61s ago 32m 69.0M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:29:16.960 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (31m) 2m ago 31m 78.4M - 17.2.0 e1d6a67b021e edb0ffb70be1 2024-04-26T16:29:16.960 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (31m) 61s ago 31m 78.9M - 17.2.0 e1d6a67b021e b6bfa2c9486b 2024-04-26T16:29:17.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:29:17.165+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:17.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:29:17.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:17.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:17.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:17.334 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:17.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:17.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:17.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:17.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:17.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:17.335 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:17.336 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:17.337 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:29:17.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:17.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:17.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:17.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:17.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:17.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:17.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:17.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:17.338 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:17.416 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:29:17.417 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:29:17.418 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-26T16:29:17.418 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 13 2024-04-26T16:29:17.418 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:29:17.418 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:29:17.792 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:29:17.792 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", 2024-04-26T16:29:17.792 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:29:17.792 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-04-26T16:29:17.793 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:29:17.793 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "", 2024-04-26T16:29:17.793 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 image", 2024-04-26T16:29:17.793 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:29:17.793 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:29:17.793 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-mon[81151]: from='client.54651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:17.793 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-mon[81151]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:17.793 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-mon[81151]: from='client.54657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:17.793 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/2678823558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:17.794 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-mon[86348]: from='client.54651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:17.794 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-mon[86348]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:17.794 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-mon[86348]: from='client.54657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:17.794 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:17 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/2678823558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:17.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:17 smithi064 ceph-mon[65271]: from='client.54651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:17.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:17 smithi064 ceph-mon[65271]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:17.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:17 smithi064 ceph-mon[65271]: from='client.54657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:17.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:17 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2678823558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:18 smithi059 ceph-mon[81151]: from='client.54663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:18 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:18.833 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:18.834 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:18.835 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:18.836 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:18.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:18.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:18.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:18.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:18.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:18.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:18.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:18.837 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:18 smithi059 ceph-mon[86348]: from='client.54663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:18 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:18.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:18.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:18 smithi059 ceph-mon[81151]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:18 smithi059 ceph-mon[86348]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:29:18.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:18 smithi064 ceph-mon[65271]: from='client.54663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:18.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:18 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:18.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:18.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:18.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:18.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:18.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:29:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:18.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:18.943 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:18.944 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:18.945 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:18.946 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:18.946 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:18.946 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:18.946 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:18.946 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:18.947 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:18 smithi064 ceph-mon[65271]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:29:19.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:19 smithi059 ceph-mon[81151]: from='client.64612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:19.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:19 smithi059 ceph-mon[81151]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:19 smithi059 ceph-mon[86348]: from='client.64612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:19 smithi059 ceph-mon[86348]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:19.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:19 smithi064 ceph-mon[65271]: from='client.64612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:19.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:19 smithi064 ceph-mon[65271]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:21.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:21 smithi064 ceph-mon[65271]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:22.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:21 smithi059 ceph-mon[86348]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:22.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:21 smithi059 ceph-mon[81151]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:29:23.623 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:23 smithi059 ceph-mon[81151]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 579 B/s rd, 0 op/s 2024-04-26T16:29:23.623 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:23 smithi059 ceph-mon[86348]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 579 B/s rd, 0 op/s 2024-04-26T16:29:23.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:23 smithi064 ceph-mon[65271]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 579 B/s rd, 0 op/s 2024-04-26T16:29:24.450 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:29:24] "GET /metrics HTTP/1.1" 200 38449 "" "Prometheus/2.43.0" 2024-04-26T16:29:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:24.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:24.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:24.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:24 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:24.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:24.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:24 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mgr 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mon 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all crash 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all osd 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mds 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:29:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:25 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:25.952 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mgr 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mon 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all crash 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all osd 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mds 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:25.953 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:29:25.954 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:29:25.954 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mgr 2024-04-26T16:29:25.954 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mon 2024-04-26T16:29:25.954 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all crash 2024-04-26T16:29:25.954 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all osd 2024-04-26T16:29:25.954 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mds 2024-04-26T16:29:25.954 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:29:25.954 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:25.954 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi059.dvtzbj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:29:25.954 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:25 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:26.904 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:26 smithi059 ceph-mon[81151]: Upgrade: Updating rgw.foo.smithi059.dvtzbj (1/2) 2024-04-26T16:29:26.904 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:26 smithi059 ceph-mon[81151]: Deploying daemon rgw.foo.smithi059.dvtzbj on smithi059 2024-04-26T16:29:26.904 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:26 smithi059 ceph-mon[86348]: Upgrade: Updating rgw.foo.smithi059.dvtzbj (1/2) 2024-04-26T16:29:26.904 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:26 smithi059 ceph-mon[86348]: Deploying daemon rgw.foo.smithi059.dvtzbj on smithi059 2024-04-26T16:29:26.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:26 smithi064 ceph-mon[65271]: Upgrade: Updating rgw.foo.smithi059.dvtzbj (1/2) 2024-04-26T16:29:26.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:26 smithi064 ceph-mon[65271]: Deploying daemon rgw.foo.smithi059.dvtzbj on smithi059 2024-04-26T16:29:27.818 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:27 smithi059 ceph-mon[81151]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:27.818 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:27 smithi059 ceph-mon[86348]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:27.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:27 smithi064 ceph-mon[65271]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:29:29.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:29 smithi059 ceph-mon[81151]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:29.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:29 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:29.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:29 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:29 smithi059 ceph-mon[86348]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:29 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:29 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:29.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:29 smithi064 ceph-mon[65271]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:29:29.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:29 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:29.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:29 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:31.906 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:31 smithi064 ceph-mon[65271]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T16:29:31.906 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:31 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:31.906 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:31 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:31.907 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:31 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:29:31.907 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:31 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:32.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:31 smithi059 ceph-mon[81151]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T16:29:32.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:31 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:32.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:31 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:32.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:31 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:29:32.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:31 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:32.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:31 smithi059 ceph-mon[86348]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T16:29:32.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:31 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:32.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:31 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:32.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:31 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi064.sqspir", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-26T16:29:32.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:31 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:32.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:32 smithi064 ceph-mon[65271]: Upgrade: Updating rgw.foo.smithi064.sqspir (2/2) 2024-04-26T16:29:32.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:32 smithi064 ceph-mon[65271]: Deploying daemon rgw.foo.smithi064.sqspir on smithi064 2024-04-26T16:29:33.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:32 smithi059 ceph-mon[81151]: Upgrade: Updating rgw.foo.smithi064.sqspir (2/2) 2024-04-26T16:29:33.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:32 smithi059 ceph-mon[81151]: Deploying daemon rgw.foo.smithi064.sqspir on smithi064 2024-04-26T16:29:33.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:32 smithi059 ceph-mon[86348]: Upgrade: Updating rgw.foo.smithi064.sqspir (2/2) 2024-04-26T16:29:33.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:32 smithi059 ceph-mon[86348]: Deploying daemon rgw.foo.smithi064.sqspir on smithi064 2024-04-26T16:29:33.765 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:33 smithi064 ceph-mon[65271]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 52 op/s 2024-04-26T16:29:33.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:33 smithi059 ceph-mon[86348]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 52 op/s 2024-04-26T16:29:33.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:33 smithi059 ceph-mon[81151]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 52 op/s 2024-04-26T16:29:34.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:29:34] "GET /metrics HTTP/1.1" 200 38446 "" "Prometheus/2.43.0" 2024-04-26T16:29:34.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:34 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:35.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:34 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:35.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:34 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:35 smithi064 ceph-mon[65271]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-04-26T16:29:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:35.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:35.944 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:35 smithi059 ceph-mon[81151]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-04-26T16:29:35.944 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:35.944 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:35.944 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:35.944 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:35 smithi059 ceph-mon[86348]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-04-26T16:29:35.944 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:35.944 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:35.944 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:37.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:37 smithi064 ceph-mon[65271]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 85 B/s wr, 107 op/s 2024-04-26T16:29:38.082 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:37 smithi059 ceph-mon[86348]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 85 B/s wr, 107 op/s 2024-04-26T16:29:38.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:37 smithi059 ceph-mon[81151]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 85 B/s wr, 107 op/s 2024-04-26T16:29:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:39 smithi064 ceph-mon[65271]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 170 B/s wr, 147 op/s 2024-04-26T16:29:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:39.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:39 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:40.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:39 smithi059 ceph-mon[81151]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 170 B/s wr, 147 op/s 2024-04-26T16:29:40.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:40.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:40.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:40.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:39 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:39 smithi059 ceph-mon[86348]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 170 B/s wr, 147 op/s 2024-04-26T16:29:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:40.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:39 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:40 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.187 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:40 smithi064 ceph-mon[65271]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-26T16:29:41.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:40 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:40 smithi059 ceph-mon[81151]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-26T16:29:41.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:40 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:41.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:40 smithi059 ceph-mon[86348]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-26T16:29:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:43 smithi064 ceph-mon[65271]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 170 B/s wr, 151 op/s 2024-04-26T16:29:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:43 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[81151]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 170 B/s wr, 151 op/s 2024-04-26T16:29:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[86348]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 170 B/s wr, 151 op/s 2024-04-26T16:29:43.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:43.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:43.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:43.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:43 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:44.416 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:29:44] "GET /metrics HTTP/1.1" 200 38446 "" "Prometheus/2.43.0" 2024-04-26T16:29:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[86348]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 185 B/s wr, 127 op/s 2024-04-26T16:29:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[86348]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 113 B/s wr, 112 op/s 2024-04-26T16:29:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:44.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:44.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[81151]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 185 B/s wr, 127 op/s 2024-04-26T16:29:44.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[81151]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 113 B/s wr, 112 op/s 2024-04-26T16:29:44.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:44.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:44.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:44.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:44 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:44 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:44 smithi064 ceph-mon[65271]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 185 B/s wr, 127 op/s 2024-04-26T16:29:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:44 smithi064 ceph-mon[65271]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 113 B/s wr, 112 op/s 2024-04-26T16:29:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:44 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:44 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:44.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:44 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:45.479 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:29:45.334+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:45.479 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:29:45.479 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:45.479 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:45.479 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:45.479 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:45.479 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:45.479 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:45.480 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:45.481 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:45.481 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:45.481 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:45.481 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:45.481 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:45.481 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:45.481 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:45.481 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:45.481 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:45.481 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:45.482 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:45.483 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:45.484 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:45.484 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:45.484 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:45.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:45.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi059.dvtzbj"}]: dispatch 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi059.dvtzbj"}]': finished 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi064.sqspir"}]: dispatch 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi064.sqspir"}]': finished 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:29:45.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:45.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi059.dvtzbj"}]: dispatch 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi059.dvtzbj"}]': finished 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi064.sqspir"}]: dispatch 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi064.sqspir"}]': finished 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.836 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:29:45.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:29:45.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:29:45.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:29:45.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:29:45.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:29:45.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:29:45.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:29:45.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:29:45.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-26T16:29:45.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:29:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:29:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:29:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:29:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:29:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:29:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:29:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:45.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:29:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:29:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:45 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:45.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:45.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi059.dvtzbj"}]: dispatch 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi059.dvtzbj"}]': finished 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi064.sqspir"}]: dispatch 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi064.sqspir"}]': finished 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-26T16:29:45.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-26T16:29:45.940 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-26T16:29:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-26T16:29:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-26T16:29:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-26T16:29:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-26T16:29:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-26T16:29:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-26T16:29:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-26T16:29:45.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:45.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:45 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:46.717 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.718 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:46.719 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:46.720 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:46.721 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:46.721 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:46.721 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:46.721 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:46.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:46.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:46.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:46.721 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:46.722 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:29:46.723 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:46.724 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all rgw 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:29:46.725 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all nfs 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: Upgrade: Finalizing container_image settings 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 113 B/s wr, 81 op/s 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: Upgrade: Complete! 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 36 op/s 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1015 B/s rd, 0 op/s 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: Checking dashboard <-> RGW credentials 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:46.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all rgw 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all nfs 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: Upgrade: Finalizing container_image settings 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 113 B/s wr, 81 op/s 2024-04-26T16:29:46.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: Upgrade: Complete! 2024-04-26T16:29:46.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 36 op/s 2024-04-26T16:29:46.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1015 B/s rd, 0 op/s 2024-04-26T16:29:46.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: Checking dashboard <-> RGW credentials 2024-04-26T16:29:46.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:46.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:46.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:46.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:46.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:46.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:46.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:29:46.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all rgw 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all nfs 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all nvmeof 2024-04-26T16:29:46.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: Upgrade: Finalizing container_image settings 2024-04-26T16:29:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 113 B/s wr, 81 op/s 2024-04-26T16:29:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: Upgrade: Complete! 2024-04-26T16:29:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 36 op/s 2024-04-26T16:29:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1015 B/s rd, 0 op/s 2024-04-26T16:29:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: Checking dashboard <-> RGW credentials 2024-04-26T16:29:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:46.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:29:46.943 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:47.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:47.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:47.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:47.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:47.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:29:47.694+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:47.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:29:47.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:47.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:47.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:47.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:47.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:47.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:47.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:47.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:47.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:47.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:47.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:47 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:47.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:47 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:47.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:47 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[81151]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-04-26T16:29:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:48.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[86348]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-04-26T16:29:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:48.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:48 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:48 smithi064 ceph-mon[65271]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-04-26T16:29:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:29:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:29:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:29:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:29:48.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:48 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:29:48.942 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:29:49.751 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:49.751 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:49.751 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:49.751 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:49.751 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:49.751 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.751 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.751 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:49.751 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.752 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:29:49.753 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:49.754 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.755 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.755 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:49.755 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:49.755 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:49.755 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:49.755 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:49.755 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[81151]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[81151]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-26T16:29:49.756 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:49.757 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:49.757 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[81151]: from='client.64699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:49.757 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:49.757 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:49.757 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:49.757 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:49.757 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:49.758 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:49.759 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:49.760 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:49.761 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:49.762 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:49.762 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:49.762 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:49.762 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.762 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.762 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[86348]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-26T16:29:49.762 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[86348]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-26T16:29:49.762 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:49.762 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:49.762 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:49 smithi059 ceph-mon[86348]: from='client.64699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:49.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:49 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:49.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:49.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:49.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:49.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:49.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:49 smithi064 ceph-mon[65271]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:49 smithi064 ceph-mon[65271]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:49 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:49 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:29:49.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:49 smithi064 ceph-mon[65271]: from='client.64699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:29:50.812+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:29:51.084 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:51.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:29:51.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:51.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:51.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:51.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:51.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:51.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:51.085 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:51.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:51.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:51.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:51.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:51.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:51.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:51.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:51.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:51.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:51.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:51.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:29:51.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:51.087 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:29:51.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:51.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:29:51.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:51.088 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:29:51.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:51.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:51.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:51.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:29:51.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:51.089 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:51.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:51.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:51.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:51.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:29:51.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:51.090 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:29:51.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:51.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:29:51.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:51.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:51.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:51.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:51.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:51.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:51.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:51.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:51.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:50 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:51.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:51 smithi059 ceph-mon[81151]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 5.4 KiB/s rd, 5 op/s 2024-04-26T16:29:51.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:51 smithi059 ceph-mon[86348]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 5.4 KiB/s rd, 5 op/s 2024-04-26T16:29:51.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:51 smithi064 ceph-mon[65271]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 5.4 KiB/s rd, 5 op/s 2024-04-26T16:29:52.025 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:29:52.025 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (10m) 13s ago 32m 29.2M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:29:52.025 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (10m) 13s ago 32m 80.7M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:29:52.025 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 13s ago 31m 62.5M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:29:52.025 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (11m) 13s ago 37m 466M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:29:52.025 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (21m) 13s ago 38m 530M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (8m) 13s ago 38m 64.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (8m) 13s ago 37m 56.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (7m) 13s ago 37m 51.9M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (11m) 13s ago 33m 16.5M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (11m) 13s ago 33m 17.4M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (5m) 13s ago 36m 61.1M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (5m) 13s ago 36m 53.9M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (4m) 13s ago 35m 48.5M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (3m) 13s ago 35m 51.8M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13cc20260a 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (3m) 13s ago 35m 59.1M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd a170975ea549 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (2m) 13s ago 34m 53.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 833404a87b94 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (2m) 13s ago 34m 55.1M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd ab006709251f 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (101s) 13s ago 34m 55.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 5757a13fa26c 2024-04-26T16:29:52.026 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (10m) 13s ago 33m 73.3M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:29:52.027 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (23s) 13s ago 31m 75.2M - 19.0.0-3305-ga430f9ed 031ba2747cfd 2cfce289d1f0 2024-04-26T16:29:52.027 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (17s) 13s ago 31m 77.4M - 19.0.0-3305-ga430f9ed 031ba2747cfd 899b8337e3a4 2024-04-26T16:29:52.787 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:52 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:52.787 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:52.787 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:52.787 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:52.787 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:52.787 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.787 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.788 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:52.789 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:52.790 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:29:52.791 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:52.791 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:52.791 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:52.791 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:52.791 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:52.791 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:52.791 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:52.791 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.791 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:52 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.834 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:52.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:52.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:52.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:52.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:52.835 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:52.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:52.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:52.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:52.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:52.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:52.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:52.836 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:52.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:52.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:52.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:52.837 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:52.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:52.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:52.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:52.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:29:52.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:52.838 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:29:52.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:52.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:29:52.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:52.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:52.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:52.839 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:52.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:52.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:52.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:52.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:52.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:52.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.840 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:52.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:52.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:52.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:52.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:52.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:52.842 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:52.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.843 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:52 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:52.843 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:52.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:52.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:52.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:52.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.844 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:52.845 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:52.845 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:52.845 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:52.845 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:52.845 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:52.845 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:52.846 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:52.846 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:52.846 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:52.846 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:52.846 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:52.846 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:52.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:29:52.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:29:52.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:29:52.847 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:29:52.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:29:52.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:29:52.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:29:52.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:29:52.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:29:52.848 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:29:52.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:29:52.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:29:52.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:29:52.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:29:52.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:29:52.849 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:29:52.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:29:52.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:29:52.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:29:52.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:29:52.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:29:52.850 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:29:52.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:29:52.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:29:52.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:29:52.851 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:29:52.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:29:52.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:29:52.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:29:52.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:29:52.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:29:52.852 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:29:52.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:29:52.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:29:52.853 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:29:52.917 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-04-26T16:29:53.187 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:29:52 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=cleanup t=2024-04-26T16:29:52.787001192Z level=info msg="Completed cleanup jobs" duration=3.644096ms 2024-04-26T16:29:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:53 smithi059 ceph-mon[86348]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 4 op/s 2024-04-26T16:29:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:53 smithi059 ceph-mon[86348]: from='client.54771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:53 smithi059 ceph-mon[81151]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 4 op/s 2024-04-26T16:29:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:53 smithi059 ceph-mon[81151]: from='client.54771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:53.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:53 smithi064 ceph-mon[65271]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 4 op/s 2024-04-26T16:29:53.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:53 smithi064 ceph-mon[65271]: from='client.54771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:29:54.427 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:29:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:29:54] "GET /metrics HTTP/1.1" 200 38477 "" "Prometheus/2.43.0" 2024-04-26T16:29:54.702 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:54 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:54.702 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:54 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:54.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:54 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:29:55.149 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:29:55.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:55 smithi059 ceph-mon[86348]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-04-26T16:29:55.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:55 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/1815575713' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:55 smithi059 ceph-mon[81151]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-04-26T16:29:55.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:55 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/1815575713' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:55.903 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-04-26T16:29:55.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:55 smithi064 ceph-mon[65271]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-04-26T16:29:55.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:55 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/1815575713' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:57.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:57 smithi059 ceph-mon[81151]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-26T16:29:57.726 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:57 smithi059 ceph-mon[86348]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-26T16:29:57.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:57 smithi064 ceph-mon[65271]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-26T16:29:58.091 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)" 2024-04-26T16:29:58.781 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:58 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/4182188197' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:58.781 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:58 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/4182188197' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:58.809 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade status' 2024-04-26T16:29:58.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:58 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/4182188197' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:29:59.832 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:29:59 smithi059 ceph-mon[86348]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-04-26T16:29:59.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:29:59 smithi059 ceph-mon[81151]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-04-26T16:29:59.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:29:59 smithi064 ceph-mon[65271]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-04-26T16:30:00.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:00 smithi059 ceph-mon[81151]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:30:00.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:00 smithi059 ceph-mon[86348]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:30:00.853 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:30:00.854 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": null, 2024-04-26T16:30:00.854 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": false, 2024-04-26T16:30:00.854 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "", 2024-04-26T16:30:00.854 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:30:00.854 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": null, 2024-04-26T16:30:00.854 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "", 2024-04-26T16:30:00.854 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:30:00.854 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:30:00.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:00 smithi064 ceph-mon[65271]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:30:01.464 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:30:01.721 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:01 smithi059 ceph-mon[81151]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-04-26T16:30:01.721 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:01.721 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:01 smithi059 ceph-mon[86348]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-04-26T16:30:01.721 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:01 smithi064 ceph-mon[65271]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-04-26T16:30:01.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:02.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:02 smithi059 ceph-mon[86348]: from='client.54789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:02.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:02 smithi059 ceph-mon[81151]: from='client.54789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:02.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:02 smithi064 ceph-mon[65271]: from='client.54789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:03.663 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:30:03.663 INFO:teuthology.orchestra.run.smithi059.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:30:03.663 INFO:teuthology.orchestra.run.smithi059.stdout: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:30:03.663 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:03 smithi059 ceph-mon[81151]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:03.664 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:03 smithi059 ceph-mon[86348]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:03.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:03 smithi064 ceph-mon[65271]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:04.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:30:04] "GET /metrics HTTP/1.1" 200 38477 "" "Prometheus/2.43.0" 2024-04-26T16:30:04.434 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-26T16:30:04.755 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:04 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:04.755 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:04 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/1714752885' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:30:04.756 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:04 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:04.756 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:04 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/1714752885' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:30:04.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:04 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:04.838 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:04 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/1714752885' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:30:05.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:05 smithi059 ceph-mon[86348]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:05.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:05 smithi059 ceph-mon[81151]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:05.855 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:05 smithi064 ceph-mon[65271]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:06.513 INFO:teuthology.orchestra.run.smithi059.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:30:07.212 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T16:30:07.222 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi059.front.sepia.ceph.com 2024-04-26T16:30:07.222 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-26T16:30:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: from='client.64735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:30:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:30:07.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: from='client.64735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:30:07.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-26T16:30:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:30:07.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:07 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: from='client.64735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:30:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:07.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:07.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:07.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:30:07.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-26T16:30:07.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:07.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:07.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:30:07.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:07 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:08.785 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:08 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:30:08.785 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:08 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:30:08.785 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:08 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:30:08.786 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:08 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:30:08.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:08 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:30:08.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:08 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:30:09.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-mon[86348]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:09.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-mon[81151]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:09.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:30:09.723+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:30:09.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:30:09.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:30:09.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:30:09.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:30:09.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:09.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:09.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:30:09.834 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:09.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return future.result(timeout) 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return self.__get_result() 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: raise self._exception 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:30:09.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: use_current_daemon_image=reconfig, 2024-04-26T16:30:09.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:30:09.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:30:09.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:30:09.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Creating ceph-iscsi config... 2024-04-26T16:30:09.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:30:09.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:30:09.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:30:09.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:09.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: Traceback (most recent call last): 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: return _run_code(code, main_globals, None, 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: exec(code, run_globals) 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:30:09.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:30:09.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:09.839 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:09 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:09.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:09 smithi064 ceph-mon[65271]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:10.429 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:30:11.138 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:30:11.138 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (10m) 32s ago 32m 29.2M - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:30:11.138 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (10m) 32s ago 32m 80.7M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:30:11.138 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 unknown 32s ago 32m 62.5M - 3.5 e1d6a67b021e be1ada8f97a8 2024-04-26T16:30:11.138 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (12m) 32s ago 37m 466M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:30:11.138 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (22m) 32s ago 39m 530M - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:30:11.138 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (8m) 32s ago 39m 64.6M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (9m) 32s ago 37m 56.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (7m) 32s ago 37m 51.9M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (11m) 32s ago 33m 16.5M - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (11m) 32s ago 33m 17.4M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (6m) 32s ago 37m 61.1M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (5m) 32s ago 36m 53.9M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (4m) 32s ago 36m 48.5M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (3m) 32s ago 35m 51.8M 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13cc20260a 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (3m) 32s ago 35m 59.1M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd a170975ea549 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (2m) 32s ago 35m 53.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 833404a87b94 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (2m) 32s ago 34m 55.1M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd ab006709251f 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (2m) 32s ago 34m 55.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 5757a13fa26c 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (11m) 32s ago 33m 73.3M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:30:11.139 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (42s) 32s ago 32m 75.2M - 19.0.0-3305-ga430f9ed 031ba2747cfd 2cfce289d1f0 2024-04-26T16:30:11.140 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (36s) 32s ago 32m 77.4M - 19.0.0-3305-ga430f9ed 031ba2747cfd 899b8337e3a4 2024-04-26T16:30:11.587 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:30:11.587 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:30:11.587 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:30:11.587 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 15 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:30:11.588 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:30:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:11 smithi064 ceph-mon[65271]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:30:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:30:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:30:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:30:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:30:11.937 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:30:11.938 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return future.result(timeout) 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: return self.__get_result() 2024-04-26T16:30:11.939 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: raise self._exception 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Creating ceph-iscsi config... 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: Traceback (most recent call last): 2024-04-26T16:30:11.940 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: exec(code, run_globals) 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.941 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:11 smithi064 ceph-mon[65271]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:30:11.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:11 smithi064 ceph-mon[65271]: from='client.64741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:11.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:11 smithi064 ceph-mon[65271]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:11.942 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:11 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2193077314' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:11.950 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:30:11.951 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6", 2024-04-26T16:30:11.951 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:30:11.951 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T16:30:11.951 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [], 2024-04-26T16:30:11.951 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "", 2024-04-26T16:30:11.951 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 image", 2024-04-26T16:30:11.951 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": false 2024-04-26T16:30:11.951 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:30:11.951 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:11 smithi059 ceph-mon[81151]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:30:11.952 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:30:11.953 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:30:11.954 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:30:11.954 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:30:11.954 INFO:journalctl@ceph.mon.a.smithi059.stdout: return future.result(timeout) 2024-04-26T16:30:11.954 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:30:11.954 INFO:journalctl@ceph.mon.a.smithi059.stdout: return self.__get_result() 2024-04-26T16:30:11.954 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:30:11.954 INFO:journalctl@ceph.mon.a.smithi059.stdout: raise self._exception 2024-04-26T16:30:11.954 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:30:11.954 INFO:journalctl@ceph.mon.a.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:30:11.954 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:30:11.955 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:30:11.956 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:30:11.956 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:30:11.956 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:30:11.956 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:11 smithi059 ceph-mon[86348]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:30:11.956 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:30:11.956 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:30:11.956 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:30:11.956 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:30:11.956 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.956 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.957 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: return future.result(timeout) 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: return self.__get_result() 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-26T16:30:11.958 INFO:journalctl@ceph.mon.c.smithi059.stdout: raise self._exception 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: use_current_daemon_image=reconfig, 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: Creating ceph-iscsi config... 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: Non-zero exit code 1 from systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: Traceback (most recent call last): 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:30:11.959 INFO:journalctl@ceph.mon.c.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:30:11.960 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:30:11.960 INFO:journalctl@ceph.mon.c.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:30:11.960 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:30:11.960 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:30:11.960 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:30:11.960 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:30:11.960 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:30:11.960 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:30:11.960 INFO:journalctl@ceph.mon.a.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:30:11.960 INFO:journalctl@ceph.mon.a.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.a.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:11 smithi059 ceph-mon[81151]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:11 smithi059 ceph-mon[81151]: from='client.64741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:11 smithi059 ceph-mon[81151]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:11 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/2193077314' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1123, in deploy_daemon 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.c.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.c.smithi059.stdout: RuntimeError: Failed command: systemctl restart ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.c.smithi059.stdout: See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:11 smithi059 ceph-mon[86348]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a430f9ed84de3008ca5b632b1ed304bac0e753f6 2024-04-26T16:30:11.961 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:11 smithi059 ceph-mon[86348]: from='client.64741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:11.962 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:11 smithi059 ceph-mon[86348]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:11.962 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:11 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/2193077314' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:12.381 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T16:30:12.381 INFO:teuthology.orchestra.run.smithi059.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T16:30:12.382 INFO:teuthology.orchestra.run.smithi059.stdout: daemon iscsi.foo.smithi059.cazpww on smithi059 is in unknown state 2024-04-26T16:30:12.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='client.54813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:12.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='client.54819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:12.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:12.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:12.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:12.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:12.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:12.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/3998993304' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:12.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:12.939 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:12 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.011 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='client.54813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='client.54819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/3998993304' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:30:13.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='client.54813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='client.54819 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.013 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/3998993304' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.014 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.015 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.015 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.015 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.015 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.015 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:13.015 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:12 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:13.902 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: from='client.54831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:13.970 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:30:13.970 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:30:13.970 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mgr 2024-04-26T16:30:13.970 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mon 2024-04-26T16:30:13.970 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all crash 2024-04-26T16:30:13.970 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all osd 2024-04-26T16:30:13.970 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all mds 2024-04-26T16:30:13.971 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all rgw 2024-04-26T16:30:13.971 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:30:13.971 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:30:13.971 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:30:13.971 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[81151]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 577 B/s rd, 0 op/s 2024-04-26T16:30:13.971 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: from='client.54831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:13.971 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:30:13.971 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:30:13.971 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mgr 2024-04-26T16:30:13.971 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mon 2024-04-26T16:30:13.972 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all crash 2024-04-26T16:30:13.972 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all osd 2024-04-26T16:30:13.972 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all mds 2024-04-26T16:30:13.972 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all rgw 2024-04-26T16:30:13.972 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:30:13.972 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:30:13.972 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:30:13.972 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:13 smithi059 ceph-mon[86348]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 577 B/s rd, 0 op/s 2024-04-26T16:30:13.972 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: from='client.54831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Target is version 19.0.0-3305-ga430f9ed (squid) 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, digests ['quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45'] 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mgr 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mon 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all crash 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all osd 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all mds 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all rgw 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all rbd-mirror 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: Upgrade: Setting container_image for all ceph-exporter 2024-04-26T16:30:13.973 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:13 smithi064 ceph-mon[65271]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 577 B/s rd, 0 op/s 2024-04-26T16:30:14.300 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:30:14] "GET /metrics HTTP/1.1" 200 38477 "" "Prometheus/2.43.0" 2024-04-26T16:30:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:14 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:30:14.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:14 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:15.023 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:14 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:15.023 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:14 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:15.023 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:14 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:30:15.023 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:14 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:15.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:14 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:15.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:15.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:30:15.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:14 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:15.925 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:15 smithi059 ceph-mon[86348]: Upgrade: Updating iscsi.foo.smithi059.cazpww 2024-04-26T16:30:15.925 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:15 smithi059 ceph-mon[86348]: Deploying daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:30:15.925 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:15 smithi059 ceph-mon[86348]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:15.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:15 smithi059 ceph-mon[81151]: Upgrade: Updating iscsi.foo.smithi059.cazpww 2024-04-26T16:30:15.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:15 smithi059 ceph-mon[81151]: Deploying daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:30:15.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:15 smithi059 ceph-mon[81151]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:15.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:15 smithi064 ceph-mon[65271]: Upgrade: Updating iscsi.foo.smithi059.cazpww 2024-04-26T16:30:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:15 smithi064 ceph-mon[65271]: Deploying daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:30:15.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:15 smithi064 ceph-mon[65271]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-26T16:30:16.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:17.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:17.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:17.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:17 smithi064 ceph-mon[65271]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:30:18.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:17 smithi059 ceph-mon[81151]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:30:18.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:17 smithi059 ceph-mon[86348]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-26T16:30:19.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:19 smithi064 ceph-mon[65271]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:20.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:19 smithi059 ceph-mon[86348]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:20.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:19 smithi059 ceph-mon[81151]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:21.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:21 smithi064 ceph-mon[65271]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:22.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:21 smithi059 ceph-mon[81151]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:22.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:21 smithi059 ceph-mon[86348]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:23.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:23 smithi059 ceph-mon[86348]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:23.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:23 smithi059 ceph-mon[81151]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:23.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:23 smithi064 ceph-mon[65271]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:24.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:30:24] "GET /metrics HTTP/1.1" 200 38474 "" "Prometheus/2.43.0" 2024-04-26T16:30:24.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:24 smithi064 ceph-mon[65271]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:25.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:24 smithi059 ceph-mon[81151]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:25.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:24 smithi059 ceph-mon[86348]: from='client.35054 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:25.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:25 smithi064 ceph-mon[65271]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:26.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:25 smithi059 ceph-mon[81151]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:26.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:25 smithi059 ceph-mon[86348]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:27.833 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:30:27.430+0000 7fe29827d700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed. 2024-04-26T16:30:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[81151]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:30:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:27.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:30:27.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:27.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[86348]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:27.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:27.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:27.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:27.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:30:27.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:27.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:30:27.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:27 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:27.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:27 smithi064 ceph-mon[65271]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:27 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:27 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:27 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:27 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:30:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:27 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:27 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi059.cazpww", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-26T16:30:27.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:27 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:28.748 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[86348]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed. 2024-04-26T16:30:28.748 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[86348]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-26T16:30:28.748 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[86348]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 577 B/s rd, 0 op/s 2024-04-26T16:30:28.748 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[86348]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-04-26T16:30:28.748 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[86348]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:30:28.749 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[86348]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:30:28.749 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[86348]: Health check failed: Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-26T16:30:28.749 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[81151]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed. 2024-04-26T16:30:28.749 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[81151]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-26T16:30:28.749 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[81151]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 577 B/s rd, 0 op/s 2024-04-26T16:30:28.749 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[81151]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-04-26T16:30:28.749 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[81151]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:30:28.749 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[81151]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:30:28.749 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:28 smithi059 ceph-mon[81151]: Health check failed: Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-26T16:30:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:28 smithi064 ceph-mon[65271]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed. 2024-04-26T16:30:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:28 smithi064 ceph-mon[65271]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-26T16:30:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:28 smithi064 ceph-mon[65271]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 577 B/s rd, 0 op/s 2024-04-26T16:30:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:28 smithi064 ceph-mon[65271]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-04-26T16:30:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:28 smithi064 ceph-mon[65271]: Reconfiguring iscsi.foo.smithi059.cazpww (dependencies changed)... 2024-04-26T16:30:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:28 smithi064 ceph-mon[65271]: Reconfiguring daemon iscsi.foo.smithi059.cazpww on smithi059 2024-04-26T16:30:28.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:28 smithi064 ceph-mon[65271]: Health check failed: Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-26T16:30:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:30 smithi064 ceph-mon[65271]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:30:30.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:30:30.938 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:30 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:31.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[86348]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:31.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:31.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:31.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:30:31.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:30:31.083 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:31.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[81151]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:31.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:31.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:31.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:30:31.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:30:31.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:30 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:30:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:30:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[86348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:30:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/1128816243' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:30:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/4167795734' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2019879130"}]: dispatch 2024-04-26T16:30:31.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2019879130"}]: dispatch 2024-04-26T16:30:31.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:30:31.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[81151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:30:31.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/1128816243' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:30:31.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:31.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:31.834 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/4167795734' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2019879130"}]: dispatch 2024-04-26T16:30:31.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:31 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2019879130"}]: dispatch 2024-04-26T16:30:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:31 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-26T16:30:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:31 smithi064 ceph-mon[65271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-26T16:30:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:31 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/1128816243' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-26T16:30:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:31 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:31 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:31 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/4167795734' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2019879130"}]: dispatch 2024-04-26T16:30:31.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:31 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2019879130"}]: dispatch 2024-04-26T16:30:32.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:32 smithi064 ceph-mon[65271]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 447 B/s rd, 0 op/s 2024-04-26T16:30:32.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:32 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2019879130"}]': finished 2024-04-26T16:30:32.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:32 smithi064 ceph-mon[65271]: osdmap e149: 8 total, 8 up, 8 in 2024-04-26T16:30:32.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:32 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/508510844' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3081737432"}]: dispatch 2024-04-26T16:30:33.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:32 smithi059 ceph-mon[81151]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 447 B/s rd, 0 op/s 2024-04-26T16:30:33.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:32 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2019879130"}]': finished 2024-04-26T16:30:33.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:32 smithi059 ceph-mon[81151]: osdmap e149: 8 total, 8 up, 8 in 2024-04-26T16:30:33.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:32 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/508510844' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3081737432"}]: dispatch 2024-04-26T16:30:33.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:32 smithi059 ceph-mon[86348]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 447 B/s rd, 0 op/s 2024-04-26T16:30:33.051 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:32 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2019879130"}]': finished 2024-04-26T16:30:33.051 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:32 smithi059 ceph-mon[86348]: osdmap e149: 8 total, 8 up, 8 in 2024-04-26T16:30:33.051 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:32 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/508510844' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3081737432"}]: dispatch 2024-04-26T16:30:34.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:30:34] "GET /metrics HTTP/1.1" 200 38555 "" "Prometheus/2.43.0" 2024-04-26T16:30:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:34 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/508510844' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3081737432"}]': finished 2024-04-26T16:30:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:34 smithi059 ceph-mon[86348]: osdmap e150: 8 total, 8 up, 8 in 2024-04-26T16:30:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:34 smithi059 ceph-mon[86348]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:30:34.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:34 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/1634186825' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2792523227"}]: dispatch 2024-04-26T16:30:34.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:34 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/508510844' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3081737432"}]': finished 2024-04-26T16:30:34.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:34 smithi059 ceph-mon[81151]: osdmap e150: 8 total, 8 up, 8 in 2024-04-26T16:30:34.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:34 smithi059 ceph-mon[81151]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:30:34.334 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:34 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/1634186825' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2792523227"}]: dispatch 2024-04-26T16:30:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:34 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/508510844' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3081737432"}]': finished 2024-04-26T16:30:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:34 smithi064 ceph-mon[65271]: osdmap e150: 8 total, 8 up, 8 in 2024-04-26T16:30:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:34 smithi064 ceph-mon[65271]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-26T16:30:34.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:34 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/1634186825' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2792523227"}]: dispatch 2024-04-26T16:30:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:35 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/1634186825' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2792523227"}]': finished 2024-04-26T16:30:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:35 smithi064 ceph-mon[65271]: osdmap e151: 8 total, 8 up, 8 in 2024-04-26T16:30:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:30:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:35 smithi064 ceph-mon[65271]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 645 B/s rd, 0 op/s 2024-04-26T16:30:35.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:35.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:35 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/1148475744' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3678339390"}]: dispatch 2024-04-26T16:30:35.438 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:35 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3678339390"}]: dispatch 2024-04-26T16:30:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/1634186825' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2792523227"}]': finished 2024-04-26T16:30:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[81151]: osdmap e151: 8 total, 8 up, 8 in 2024-04-26T16:30:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:30:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[81151]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 645 B/s rd, 0 op/s 2024-04-26T16:30:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/1148475744' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3678339390"}]: dispatch 2024-04-26T16:30:35.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3678339390"}]: dispatch 2024-04-26T16:30:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/1634186825' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/2792523227"}]': finished 2024-04-26T16:30:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[86348]: osdmap e151: 8 total, 8 up, 8 in 2024-04-26T16:30:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:30:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:30:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[86348]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 645 B/s rd, 0 op/s 2024-04-26T16:30:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/1148475744' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3678339390"}]: dispatch 2024-04-26T16:30:35.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:35 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3678339390"}]: dispatch 2024-04-26T16:30:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:36 smithi064 ceph-mon[65271]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T16:30:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:36 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3678339390"}]': finished 2024-04-26T16:30:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:36 smithi064 ceph-mon[65271]: osdmap e152: 8 total, 8 up, 8 in 2024-04-26T16:30:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:36 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/1297068367' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/273056029"}]: dispatch 2024-04-26T16:30:36.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:36 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/273056029"}]: dispatch 2024-04-26T16:30:36.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:36 smithi059 ceph-mon[81151]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T16:30:36.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:36 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3678339390"}]': finished 2024-04-26T16:30:36.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:36 smithi059 ceph-mon[81151]: osdmap e152: 8 total, 8 up, 8 in 2024-04-26T16:30:36.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:36 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/1297068367' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/273056029"}]: dispatch 2024-04-26T16:30:36.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:36 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/273056029"}]: dispatch 2024-04-26T16:30:36.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:36 smithi059 ceph-mon[86348]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T16:30:36.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:36 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3678339390"}]': finished 2024-04-26T16:30:36.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:36 smithi059 ceph-mon[86348]: osdmap e152: 8 total, 8 up, 8 in 2024-04-26T16:30:36.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:36 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/1297068367' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/273056029"}]: dispatch 2024-04-26T16:30:36.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:36 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/273056029"}]: dispatch 2024-04-26T16:30:37.582 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:37 smithi059 ceph-mon[86348]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 749 B/s rd, 0 op/s 2024-04-26T16:30:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:37 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/273056029"}]': finished 2024-04-26T16:30:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:37 smithi059 ceph-mon[86348]: osdmap e153: 8 total, 8 up, 8 in 2024-04-26T16:30:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:37 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/775487377' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3161387799"}]: dispatch 2024-04-26T16:30:37.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:37 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3161387799"}]: dispatch 2024-04-26T16:30:37.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:37 smithi059 ceph-mon[81151]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 749 B/s rd, 0 op/s 2024-04-26T16:30:37.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:37 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/273056029"}]': finished 2024-04-26T16:30:37.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:37 smithi059 ceph-mon[81151]: osdmap e153: 8 total, 8 up, 8 in 2024-04-26T16:30:37.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:37 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/775487377' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3161387799"}]: dispatch 2024-04-26T16:30:37.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:37 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3161387799"}]: dispatch 2024-04-26T16:30:37.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:37 smithi064 ceph-mon[65271]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 749 B/s rd, 0 op/s 2024-04-26T16:30:37.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:37 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/273056029"}]': finished 2024-04-26T16:30:37.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:37 smithi064 ceph-mon[65271]: osdmap e153: 8 total, 8 up, 8 in 2024-04-26T16:30:37.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:37 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/775487377' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3161387799"}]: dispatch 2024-04-26T16:30:37.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:37 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3161387799"}]: dispatch 2024-04-26T16:30:38.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:38 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3161387799"}]': finished 2024-04-26T16:30:38.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:38 smithi059 ceph-mon[81151]: osdmap e154: 8 total, 8 up, 8 in 2024-04-26T16:30:38.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:38 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/3952655443' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1337714491"}]: dispatch 2024-04-26T16:30:38.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:38 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1337714491"}]: dispatch 2024-04-26T16:30:38.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:38 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3161387799"}]': finished 2024-04-26T16:30:38.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:38 smithi059 ceph-mon[86348]: osdmap e154: 8 total, 8 up, 8 in 2024-04-26T16:30:38.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:38 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/3952655443' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1337714491"}]: dispatch 2024-04-26T16:30:38.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:38 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1337714491"}]: dispatch 2024-04-26T16:30:38.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:38 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3161387799"}]': finished 2024-04-26T16:30:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:38 smithi064 ceph-mon[65271]: osdmap e154: 8 total, 8 up, 8 in 2024-04-26T16:30:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:38 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/3952655443' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1337714491"}]: dispatch 2024-04-26T16:30:38.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:38 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1337714491"}]: dispatch 2024-04-26T16:30:39.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:39 smithi059 ceph-mon[81151]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:39.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:39 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1337714491"}]': finished 2024-04-26T16:30:39.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:39 smithi059 ceph-mon[81151]: osdmap e155: 8 total, 8 up, 8 in 2024-04-26T16:30:39.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:39 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/833818022' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1519873150"}]: dispatch 2024-04-26T16:30:39.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:39 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1519873150"}]: dispatch 2024-04-26T16:30:39.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:39 smithi059 ceph-mon[86348]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:39.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:39 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1337714491"}]': finished 2024-04-26T16:30:39.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:39 smithi059 ceph-mon[86348]: osdmap e155: 8 total, 8 up, 8 in 2024-04-26T16:30:39.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:39 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/833818022' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1519873150"}]: dispatch 2024-04-26T16:30:39.585 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:39 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1519873150"}]: dispatch 2024-04-26T16:30:39.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:39 smithi064 ceph-mon[65271]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:39.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:39 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1337714491"}]': finished 2024-04-26T16:30:39.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:39 smithi064 ceph-mon[65271]: osdmap e155: 8 total, 8 up, 8 in 2024-04-26T16:30:39.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:39 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/833818022' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1519873150"}]: dispatch 2024-04-26T16:30:39.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:39 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1519873150"}]: dispatch 2024-04-26T16:30:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:40 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1519873150"}]': finished 2024-04-26T16:30:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:40 smithi059 ceph-mon[81151]: osdmap e156: 8 total, 8 up, 8 in 2024-04-26T16:30:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:40 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/2765598782' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1531579567"}]: dispatch 2024-04-26T16:30:40.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:40 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1531579567"}]: dispatch 2024-04-26T16:30:40.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:40 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1519873150"}]': finished 2024-04-26T16:30:40.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:40 smithi059 ceph-mon[86348]: osdmap e156: 8 total, 8 up, 8 in 2024-04-26T16:30:40.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:40 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/2765598782' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1531579567"}]: dispatch 2024-04-26T16:30:40.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:40 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1531579567"}]: dispatch 2024-04-26T16:30:40.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:40 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6801/1519873150"}]': finished 2024-04-26T16:30:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:40 smithi064 ceph-mon[65271]: osdmap e156: 8 total, 8 up, 8 in 2024-04-26T16:30:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:40 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2765598782' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1531579567"}]: dispatch 2024-04-26T16:30:40.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:40 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1531579567"}]: dispatch 2024-04-26T16:30:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[81151]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1531579567"}]': finished 2024-04-26T16:30:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[81151]: osdmap e157: 8 total, 8 up, 8 in 2024-04-26T16:30:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/3987599316' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/273056029"}]: dispatch 2024-04-26T16:30:41.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/273056029"}]: dispatch 2024-04-26T16:30:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[86348]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:41.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1531579567"}]': finished 2024-04-26T16:30:41.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[86348]: osdmap e157: 8 total, 8 up, 8 in 2024-04-26T16:30:41.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:41.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/3987599316' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/273056029"}]: dispatch 2024-04-26T16:30:41.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:41 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/273056029"}]: dispatch 2024-04-26T16:30:41.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:41 smithi064 ceph-mon[65271]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:41.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:41 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/1531579567"}]': finished 2024-04-26T16:30:41.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:41 smithi064 ceph-mon[65271]: osdmap e157: 8 total, 8 up, 8 in 2024-04-26T16:30:41.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:41 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:41.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:41 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/3987599316' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/273056029"}]: dispatch 2024-04-26T16:30:41.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:41 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/273056029"}]: dispatch 2024-04-26T16:30:42.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:42 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/273056029"}]': finished 2024-04-26T16:30:42.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:42 smithi059 ceph-mon[86348]: osdmap e158: 8 total, 8 up, 8 in 2024-04-26T16:30:42.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:42 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/260310502' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1519873150"}]: dispatch 2024-04-26T16:30:42.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:42 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1519873150"}]: dispatch 2024-04-26T16:30:42.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:42 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/273056029"}]': finished 2024-04-26T16:30:42.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:42 smithi059 ceph-mon[81151]: osdmap e158: 8 total, 8 up, 8 in 2024-04-26T16:30:42.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:42 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/260310502' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1519873150"}]: dispatch 2024-04-26T16:30:42.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:42 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1519873150"}]: dispatch 2024-04-26T16:30:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:42 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/273056029"}]': finished 2024-04-26T16:30:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:42 smithi064 ceph-mon[65271]: osdmap e158: 8 total, 8 up, 8 in 2024-04-26T16:30:42.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:42 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/260310502' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1519873150"}]: dispatch 2024-04-26T16:30:42.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:42 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1519873150"}]: dispatch 2024-04-26T16:30:42.796 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:30:43.151 INFO:teuthology.orchestra.run.smithi059.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed." 2024-04-26T16:30:43.493 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:43 smithi059 ceph-mon[81151]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:30:43.493 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:43 smithi059 ceph-mon[81151]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1519873150"}]': finished 2024-04-26T16:30:43.493 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:43 smithi059 ceph-mon[81151]: osdmap e159: 8 total, 8 up, 8 in 2024-04-26T16:30:43.493 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:43 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/2822709987' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3342820479"}]: dispatch 2024-04-26T16:30:43.493 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:43 smithi059 ceph-mon[86348]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:30:43.494 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:43 smithi059 ceph-mon[86348]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1519873150"}]': finished 2024-04-26T16:30:43.494 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:43 smithi059 ceph-mon[86348]: osdmap e159: 8 total, 8 up, 8 in 2024-04-26T16:30:43.494 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:43 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/2822709987' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3342820479"}]: dispatch 2024-04-26T16:30:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:43 smithi064 ceph-mon[65271]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-04-26T16:30:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:43 smithi064 ceph-mon[65271]: from='client.? ' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:6800/1519873150"}]': finished 2024-04-26T16:30:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:43 smithi064 ceph-mon[65271]: osdmap e159: 8 total, 8 up, 8 in 2024-04-26T16:30:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:43 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2822709987' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3342820479"}]: dispatch 2024-04-26T16:30:43.897 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:30:44.547 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:44 smithi059 ceph-mon[81151]: from='client.54939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:44.560 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:44 smithi059 ceph-mon[81151]: from='client.54945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:44.560 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:44 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/2822709987' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3342820479"}]': finished 2024-04-26T16:30:44.560 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:44 smithi059 ceph-mon[81151]: osdmap e160: 8 total, 8 up, 8 in 2024-04-26T16:30:44.560 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:30:44] "GET /metrics HTTP/1.1" 200 38555 "" "Prometheus/2.43.0" 2024-04-26T16:30:44.561 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:44 smithi059 ceph-mon[86348]: from='client.54939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:44.561 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:44 smithi059 ceph-mon[86348]: from='client.54945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:44.561 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:44 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/2822709987' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3342820479"}]': finished 2024-04-26T16:30:44.561 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:44 smithi059 ceph-mon[86348]: osdmap e160: 8 total, 8 up, 8 in 2024-04-26T16:30:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:44 smithi064 ceph-mon[65271]: from='client.54939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:44 smithi064 ceph-mon[65271]: from='client.54945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:44 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/2822709987' entity='client.iscsi.foo.smithi059.cazpww' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.59:0/3342820479"}]': finished 2024-04-26T16:30:44.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:44 smithi064 ceph-mon[65271]: osdmap e160: 8 total, 8 up, 8 in 2024-04-26T16:30:45.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:45 smithi059 ceph-mon[86348]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:45.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:45 smithi059 ceph-mon[81151]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:45.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:45 smithi064 ceph-mon[65271]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:46.067 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:30:46.067 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (11m) 11s ago 33m - - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:30:46.067 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (10m) 67s ago 33m 80.7M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:30:46.067 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 running (16s) 11s ago 32m - - 3.8 031ba2747cfd 524cf14d7e13 2024-04-26T16:30:46.067 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (12m) 67s ago 37m 466M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:30:46.067 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (22m) 11s ago 39m - - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (8m) 11s ago 39m - 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (9m) 67s ago 38m 56.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (8m) 11s ago 38m - 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (12m) 11s ago 34m - - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (12m) 67s ago 34m 17.4M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (6m) 11s ago 37m - 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (6m) 11s ago 37m - 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (5m) 11s ago 36m - 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (4m) 11s ago 36m - 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13cc20260a 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (3m) 67s ago 36m 59.1M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd a170975ea549 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (3m) 67s ago 35m 53.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 833404a87b94 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (2m) 67s ago 35m 55.1M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd ab006709251f 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (2m) 67s ago 34m 55.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 5757a13fa26c 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (11m) 67s ago 34m 73.3M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:30:46.068 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (77s) 11s ago 32m - - 19.0.0-3305-ga430f9ed 031ba2747cfd 2cfce289d1f0 2024-04-26T16:30:46.069 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (71s) 67s ago 32m 77.4M - 19.0.0-3305-ga430f9ed 031ba2747cfd 899b8337e3a4 2024-04-26T16:30:46.744 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions' 2024-04-26T16:30:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:47 smithi059 ceph-mon[86348]: from='client.54951 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:47 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:47 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:47.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:47 smithi059 ceph-mon[86348]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-04-26T16:30:47.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:47 smithi059 ceph-mon[81151]: from='client.54951 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:47.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:47 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:47.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:47 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:47.333 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:47 smithi059 ceph-mon[81151]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-04-26T16:30:47.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:47 smithi064 ceph-mon[65271]: from='client.54951 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:30:47.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:47 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:30:47.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:47 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:30:47.437 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:47 smithi064 ceph-mon[65271]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-04-26T16:30:48.808 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:30:48.808 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:30:48.808 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 15 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:30:48.809 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:30:49.443 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:49 smithi059 ceph-mon[81151]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-26T16:30:49.443 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:49 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/1015243697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:49.443 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:49 smithi059 ceph-mon[86348]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-26T16:30:49.443 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:49 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/1015243697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:49.467 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-26T16:30:49.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:49 smithi064 ceph-mon[65271]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-26T16:30:49.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:49 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/1015243697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:30:51.254 INFO:teuthology.orchestra.run.smithi059.stdout:wait for servicemap items w/ changing names to refresh 2024-04-26T16:30:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:51 smithi059 ceph-mon[86348]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:51.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:51 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:51 smithi059 ceph-mon[81151]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:51.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:51 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:51 smithi064 ceph-mon[65271]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:51 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:30:52.055 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'sleep 60' 2024-04-26T16:30:53.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:53 smithi059 ceph-mon[81151]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:30:53.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:53 smithi059 ceph-mon[86348]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:30:53.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:53 smithi064 ceph-mon[65271]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-26T16:30:54.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:30:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:30:54] "GET /metrics HTTP/1.1" 200 38556 "" "Prometheus/2.43.0" 2024-04-26T16:30:55.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:55 smithi059 ceph-mon[86348]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-04-26T16:30:55.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:55 smithi059 ceph-mon[81151]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-04-26T16:30:55.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:55 smithi064 ceph-mon[65271]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 935 B/s rd, 0 op/s 2024-04-26T16:30:57.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:57 smithi059 ceph-mon[86348]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:57.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:57 smithi059 ceph-mon[81151]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:57 smithi064 ceph-mon[65271]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:30:59.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:30:59 smithi059 ceph-mon[81151]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:59.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:30:59 smithi059 ceph-mon[86348]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:30:59.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:30:59 smithi064 ceph-mon[65271]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:01.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:01 smithi059 ceph-mon[81151]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:01.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:01 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:01.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:01.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:01 smithi059 ceph-mon[86348]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:01.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:01 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:01.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:01.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:01 smithi064 ceph-mon[65271]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:01.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:01 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:01.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:03.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:03 smithi059 ceph-mon[81151]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:03 smithi059 ceph-mon[86348]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:03 smithi064 ceph-mon[65271]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:04.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:31:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:31:04] "GET /metrics HTTP/1.1" 200 38556 "" "Prometheus/2.43.0" 2024-04-26T16:31:05.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:05 smithi059 ceph-mon[86348]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:05.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:05 smithi059 ceph-mon[81151]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:05 smithi064 ceph-mon[65271]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:07.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:07 smithi059 ceph-mon[86348]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:07.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:07 smithi059 ceph-mon[81151]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:07.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:07 smithi064 ceph-mon[65271]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:09.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:09 smithi059 ceph-mon[86348]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:09.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:09 smithi059 ceph-mon[81151]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:09.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:09 smithi064 ceph-mon[65271]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:11 smithi059 ceph-mon[86348]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:11.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:11 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:11 smithi059 ceph-mon[81151]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:11.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:11 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:11 smithi064 ceph-mon[65271]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:11 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:13 smithi059 ceph-mon[81151]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:13.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:13 smithi059 ceph-mon[86348]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:13 smithi064 ceph-mon[65271]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:14.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:31:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:31:14] "GET /metrics HTTP/1.1" 200 38556 "" "Prometheus/2.43.0" 2024-04-26T16:31:15.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:15 smithi059 ceph-mon[81151]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:15.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:15 smithi059 ceph-mon[86348]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:15.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:15 smithi064 ceph-mon[65271]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:16.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:16.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:17.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:17 smithi059 ceph-mon[81151]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:17.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:17 smithi059 ceph-mon[86348]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:17 smithi064 ceph-mon[65271]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:19.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:19 smithi059 ceph-mon[81151]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:19.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:19 smithi059 ceph-mon[86348]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:19 smithi064 ceph-mon[65271]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:21.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:21 smithi059 ceph-mon[81151]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:21.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:21 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:21 smithi059 ceph-mon[86348]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:21.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:21 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:21 smithi064 ceph-mon[65271]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:21.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:21 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:23.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:23 smithi059 ceph-mon[81151]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:23.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:23 smithi059 ceph-mon[86348]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:23.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:23 smithi064 ceph-mon[65271]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:24.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:31:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:31:24] "GET /metrics HTTP/1.1" 200 38555 "" "Prometheus/2.43.0" 2024-04-26T16:31:25.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:25 smithi059 ceph-mon[81151]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:25.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:25 smithi059 ceph-mon[86348]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:25.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:25 smithi064 ceph-mon[65271]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:27.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:27 smithi059 ceph-mon[86348]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:27.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:27 smithi059 ceph-mon[81151]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:27.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:27 smithi064 ceph-mon[65271]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:29.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:29 smithi064 ceph-mon[65271]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:29.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:29 smithi059 ceph-mon[86348]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:29.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:29 smithi059 ceph-mon[81151]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:31.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:31 smithi064 ceph-mon[65271]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:31.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:31 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:31.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:31 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:31.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:31 smithi059 ceph-mon[81151]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:31.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:31 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:31.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:31 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:31 smithi059 ceph-mon[86348]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:31 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:31.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:31 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:33.635 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:33 smithi059 ceph-mon[81151]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:33.635 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:33 smithi059 ceph-mon[86348]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:33.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:33 smithi064 ceph-mon[65271]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:34.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:31:34 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:31:34] "GET /metrics HTTP/1.1" 200 38555 "" "Prometheus/2.43.0" 2024-04-26T16:31:35.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:35 smithi064 ceph-mon[65271]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:35.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:31:35.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:31:35.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:31:35.688 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:35 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:31:35.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:35 smithi059 ceph-mon[81151]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:35.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:31:35.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:31:35.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:31:35.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:35 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:31:35.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:35 smithi059 ceph-mon[86348]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:35.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T16:31:35.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T16:31:35.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T16:31:35.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:35 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' 2024-04-26T16:31:37.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:37 smithi064 ceph-mon[65271]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:37.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:37 smithi059 ceph-mon[81151]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:37.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:37 smithi059 ceph-mon[86348]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:39.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:39 smithi064 ceph-mon[65271]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:39.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:39 smithi059 ceph-mon[81151]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:39.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:39 smithi059 ceph-mon[86348]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:41.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:41 smithi064 ceph-mon[65271]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:41.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:41 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:41 smithi059 ceph-mon[81151]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:41.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:41 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:41 smithi059 ceph-mon[86348]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:41.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:41 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:43.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:43 smithi064 ceph-mon[65271]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:43.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:43 smithi059 ceph-mon[81151]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:43.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:43 smithi059 ceph-mon[86348]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-26T16:31:44.583 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:31:44 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:31:44] "GET /metrics HTTP/1.1" 200 38555 "" "Prometheus/2.43.0" 2024-04-26T16:31:45.646 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:45 smithi059 ceph-mon[81151]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:45.647 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:45 smithi059 ceph-mon[86348]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:45.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:45 smithi064 ceph-mon[65271]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:46.436 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:46 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:46.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:46 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:46.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:46 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:31:47.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:47 smithi064 ceph-mon[65271]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:47.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:47 smithi059 ceph-mon[86348]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:47.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:47 smithi059 ceph-mon[81151]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:49.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:49 smithi064 ceph-mon[65271]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:49.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:49 smithi059 ceph-mon[86348]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:49.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:49 smithi059 ceph-mon[81151]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:51 smithi064 ceph-mon[65271]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:51.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:51 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:51.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:51 smithi059 ceph-mon[81151]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:51.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:51 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:51.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:51 smithi059 ceph-mon[86348]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:51.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:51 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:31:53.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:53 smithi064 ceph-mon[65271]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:53.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:53 smithi059 ceph-mon[81151]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:53.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:53 smithi059 ceph-mon[86348]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:54.488 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:31:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:31:54] "GET /metrics HTTP/1.1" 200 38554 "" "Prometheus/2.43.0" 2024-04-26T16:31:54.520 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ps' 2024-04-26T16:31:55.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:55 smithi064 ceph-mon[65271]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:55.797 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:55 smithi059 ceph-mon[81151]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:55.798 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:55 smithi059 ceph-mon[86348]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:56.524 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T16:31:56.524 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 *:9093,9094 running (12m) 82s ago 34m - - 0.25.0 c8568f914cd2 70f919777053 2024-04-26T16:31:56.524 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi064 *:3000 running (12m) 2m ago 34m 80.7M - 9.4.12 2bacad6d85d8 5d3c77d009e2 2024-04-26T16:31:56.524 INFO:teuthology.orchestra.run.smithi059.stdout:iscsi.foo.smithi059.cazpww smithi059 running (86s) 82s ago 33m - - 3.8 031ba2747cfd 524cf14d7e13 2024-04-26T16:31:56.524 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi064 *:8443,9283,8765 running (13m) 2m ago 39m 466M - 19.0.0-3305-ga430f9ed 031ba2747cfd b30a628a06dc 2024-04-26T16:31:56.524 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443,9283,8765 running (23m) 82s ago 40m - - 19.0.0-3305-ga430f9ed 031ba2747cfd c5d6af75064d 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (10m) 82s ago 40m - 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd a406f6807e94 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi064 running (10m) 2m ago 39m 56.3M 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 71107a400e68 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (9m) 82s ago 39m - 2048M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13fa03c2b8 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 *:9100 running (13m) 82s ago 35m - - 1.5.0 0da6a335fe13 feeb6104cfe1 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi064 *:9100 running (13m) 2m ago 35m 17.4M - 1.5.0 0da6a335fe13 2d4920f76a2e 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (7m) 82s ago 38m - 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 28dcbd084295 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (7m) 82s ago 38m - 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 7e91e2761521 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (6m) 82s ago 38m - 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd e0a6538d2331 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (5m) 82s ago 37m - 2993M 19.0.0-3305-ga430f9ed 031ba2747cfd 0c13cc20260a 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi064 running (5m) 2m ago 37m 59.1M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd a170975ea549 2024-04-26T16:31:56.525 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi064 running (4m) 2m ago 36m 53.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 833404a87b94 2024-04-26T16:31:56.526 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi064 running (4m) 2m ago 36m 55.1M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd ab006709251f 2024-04-26T16:31:56.526 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi064 running (3m) 2m ago 36m 55.4M 3248M 19.0.0-3305-ga430f9ed 031ba2747cfd 5757a13fa26c 2024-04-26T16:31:56.526 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi064 *:9095 running (12m) 2m ago 35m 73.3M - 2.43.0 a07b618ecd1d 279642636cd0 2024-04-26T16:31:56.526 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi059.dvtzbj smithi059 *:8000 running (2m) 82s ago 33m - - 19.0.0-3305-ga430f9ed 031ba2747cfd 2cfce289d1f0 2024-04-26T16:31:56.526 INFO:teuthology.orchestra.run.smithi059.stdout:rgw.foo.smithi064.sqspir smithi064 *:8000 running (2m) 2m ago 33m 77.4M - 19.0.0-3305-ga430f9ed 031ba2747cfd 899b8337e3a4 2024-04-26T16:31:57.177 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions' 2024-04-26T16:31:57.664 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:57 smithi059 ceph-mon[81151]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:57.665 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:57 smithi059 ceph-mon[81151]: from='client.54963 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:31:57.665 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:57 smithi059 ceph-mon[86348]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:57.665 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:57 smithi059 ceph-mon[86348]: from='client.54963 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:31:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:57 smithi064 ceph-mon[65271]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:31:57.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:57 smithi064 ceph-mon[65271]: from='client.54963 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:31:59.215 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:31:59.215 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2024-04-26T16:31:59.215 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 3 2024-04-26T16:31:59.216 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:31:59.216 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2024-04-26T16:31:59.216 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:31:59.216 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:31:59.216 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2024-04-26T16:31:59.216 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 8 2024-04-26T16:31:59.217 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:31:59.217 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw": { 2024-04-26T16:31:59.217 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 2 2024-04-26T16:31:59.217 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2024-04-26T16:31:59.217 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2024-04-26T16:31:59.217 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)": 15 2024-04-26T16:31:59.217 INFO:teuthology.orchestra.run.smithi059.stdout: } 2024-04-26T16:31:59.218 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:31:59.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:59 smithi059 ceph-mon[81151]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:59.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:31:59 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/774531829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:31:59.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:59 smithi059 ceph-mon[86348]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:59.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:31:59 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/774531829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:31:59.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:59 smithi064 ceph-mon[65271]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:31:59.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:31:59 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/774531829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:31:59.831 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch upgrade status' 2024-04-26T16:32:01.689 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:01 smithi064 ceph-mon[65271]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:01 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:32:01.690 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:01 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:32:01.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:01 smithi059 ceph-mon[81151]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:01.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:01 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:32:01.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:01 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:32:01.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:01 smithi059 ceph-mon[86348]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:01 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:32:01.834 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:01 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:32:01.837 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:32:01.837 INFO:teuthology.orchestra.run.smithi059.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45", 2024-04-26T16:32:01.837 INFO:teuthology.orchestra.run.smithi059.stdout: "in_progress": true, 2024-04-26T16:32:01.837 INFO:teuthology.orchestra.run.smithi059.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout: "services_complete": [ 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout: "osd", 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr", 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout: "rgw", 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout: "iscsi", 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout: "mon" 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout: ], 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout: "progress": "16/21 daemons upgraded", 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed.", 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout: "is_paused": true 2024-04-26T16:32:01.838 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:32:02.612 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph health detail' 2024-04-26T16:32:03.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:03 smithi059 ceph-mon[81151]: from='client.54975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:03.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:03 smithi059 ceph-mon[81151]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:03 smithi059 ceph-mon[86348]: from='client.54975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:03.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:03 smithi059 ceph-mon[86348]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:03 smithi064 ceph-mon[65271]: from='client.54975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:03.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:03 smithi064 ceph-mon[65271]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:04.301 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:04 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:32:04] "GET /metrics HTTP/1.1" 200 38553 "" "Prometheus/2.43.0" 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed. 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi059.cazpww on host smithi059 failed. 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout: Upgrade daemon: iscsi.foo.smithi059.cazpww: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi059.cazpww ... 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:Creating ceph-iscsi config... 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/iscsi-gateway.cfg 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:Write file: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/iscsi.foo.smithi059.cazpww/tcmu-runner-entrypoint.sh 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2db6607c98\x2d03e4\x2d11ef\x2dbc93\x2dc7b262605968.slice/ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:Non-zero exit code 1 from systemctl start ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:systemctl: stderr Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:systemctl: stderr See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:32:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:Traceback (most recent call last): 2024-04-26T16:32:04.727 INFO:teuthology.orchestra.run.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-26T16:32:04.727 INFO:teuthology.orchestra.run.smithi059.stdout: return _run_code(code, main_globals, None, 2024-04-26T16:32:04.727 INFO:teuthology.orchestra.run.smithi059.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-26T16:32:04.727 INFO:teuthology.orchestra.run.smithi059.stdout: exec(code, run_globals) 2024-04-26T16:32:04.727 INFO:teuthology.orchestra.run.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5618, in 2024-04-26T16:32:04.727 INFO:teuthology.orchestra.run.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 5606, in main 2024-04-26T16:32:04.728 INFO:teuthology.orchestra.run.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3096, in command_deploy_from 2024-04-26T16:32:04.728 INFO:teuthology.orchestra.run.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3131, in _common_deploy 2024-04-26T16:32:04.728 INFO:teuthology.orchestra.run.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 3151, in _deploy_daemon_container 2024-04-26T16:32:04.728 INFO:teuthology.orchestra.run.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1087, in deploy_daemon 2024-04-26T16:32:04.728 INFO:teuthology.orchestra.run.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/__main__.py", line 1223, in deploy_daemon_units 2024-04-26T16:32:04.728 INFO:teuthology.orchestra.run.smithi059.stdout: File "/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/cephadm.f243a5289e4390f10dd5970479cb922a468099a102f28c10b225af83f6a8700b/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-26T16:32:04.728 INFO:teuthology.orchestra.run.smithi059.stdout:RuntimeError: Failed command: systemctl start ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww: Job for ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service failed because the control process exited with error code. 2024-04-26T16:32:04.729 INFO:teuthology.orchestra.run.smithi059.stdout:See "systemctl status ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" and "journalctl -xeu ceph-b6607c98-03e4-11ef-bc93-c7b262605968@iscsi.foo.smithi059.cazpww.service" for details. 2024-04-26T16:32:05.360 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-26T16:32:05.672 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:05 smithi059 ceph-mon[81151]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:05.672 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:05 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/378512783' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:32:05.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:05 smithi059 ceph-mon[86348]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:05.673 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:05 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/378512783' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:32:05.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:05 smithi064 ceph-mon[65271]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:05.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:05 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/378512783' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-26T16:32:07.485 INFO:teuthology.orchestra.run.smithi059.stdout:true 2024-04-26T16:32:07.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:07 smithi064 ceph-mon[65271]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:07.798 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:07 smithi059 ceph-mon[81151]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:07.799 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:07 smithi059 ceph-mon[86348]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:08.137 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-26T16:32:08.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:08 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/875591687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:32:08.677 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:08 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/875591687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:32:08.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:08 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/875591687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:32:09.629 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:09 smithi059 ceph-mon[81151]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:09.629 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:09 smithi059 ceph-mon[86348]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:09.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:09 smithi064 ceph-mon[65271]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:10.160 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 19.0.0-3305-ga430f9ed (a430f9ed84de3008ca5b632b1ed304bac0e753f6) squid (dev)" 2024-04-26T16:32:10.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:10 smithi064 ceph-mon[65271]: from='client.? 172.21.15.59:0/67064271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:32:10.804 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:10 smithi059 ceph-mon[86348]: from='client.? 172.21.15.59:0/67064271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:32:10.805 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:10 smithi059 ceph-mon[81151]: from='client.? 172.21.15.59:0/67064271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T16:32:10.832 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -e sha1=a430f9ed84de3008ca5b632b1ed304bac0e753f6 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-04-26T16:32:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:11 smithi064 ceph-mon[65271]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:11.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:11 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:32:11.771 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:11 smithi059 ceph-mon[86348]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:11.771 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:11 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:32:11.772 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:11 smithi059 ceph-mon[81151]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:11.772 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:11 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:32:12.812 INFO:teuthology.orchestra.run.smithi059.stdout:osd 8 2m ago - 2024-04-26T16:32:13.410 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T16:32:13.420 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi059.front.sepia.ceph.com 2024-04-26T16:32:13.420 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-04-26T16:32:13.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:13 smithi059 ceph-mon[81151]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:13.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:13 smithi059 ceph-mon[86348]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:13.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:13 smithi064 ceph-mon[65271]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:14.333 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:14 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:32:14] "GET /metrics HTTP/1.1" 200 38553 "" "Prometheus/2.43.0" 2024-04-26T16:32:14.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:14 smithi064 ceph-mon[65271]: from='client.54999 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:14.801 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:14 smithi059 ceph-mon[86348]: from='client.54999 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:14.802 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:14 smithi059 ceph-mon[81151]: from='client.54999 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:15.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:15 smithi064 ceph-mon[65271]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:15.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:15 smithi059 ceph-mon[81151]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:15.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:15 smithi059 ceph-mon[86348]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:16.364 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2024-04-26T16:32:16.364 INFO:teuthology.orchestra.run.smithi059.stdout: "image": "quay.io/ceph/ceph", 2024-04-26T16:32:16.364 INFO:teuthology.orchestra.run.smithi059.stdout: "registry": "quay.io", 2024-04-26T16:32:16.365 INFO:teuthology.orchestra.run.smithi059.stdout: "bare_image": "ceph/ceph", 2024-04-26T16:32:16.365 INFO:teuthology.orchestra.run.smithi059.stdout: "versions": [] 2024-04-26T16:32:16.365 INFO:teuthology.orchestra.run.smithi059.stdout:} 2024-04-26T16:32:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:16 smithi064 ceph-mon[65271]: from='client.55005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:16.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:16 smithi064 ceph-mon[65271]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:32:16.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:16 smithi059 ceph-mon[86348]: from='client.55005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:16.727 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:16 smithi059 ceph-mon[86348]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:32:16.727 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:16 smithi059 ceph-mon[81151]: from='client.55005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:16.727 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:16 smithi059 ceph-mon[81151]: from='mgr.54108 172.21.15.59:0/1502045572' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T16:32:17.006 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-04-26T16:32:17.578 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:17 smithi059 ceph-mon[86348]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:17.579 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:17 smithi059 ceph-mon[81151]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:17.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:17 smithi064 ceph-mon[65271]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:19.676 INFO:teuthology.orchestra.run.smithi059.stdout: "16.2.0", 2024-04-26T16:32:19.687 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:19 smithi064 ceph-mon[65271]: pgmap v406: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:19.833 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:19 smithi059 ceph-mon[81151]: pgmap v406: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:19.833 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:19 smithi059 ceph-mon[86348]: pgmap v406: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-26T16:32:20.345 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b6607c98-03e4-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-04-26T16:32:20.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:20 smithi059 ceph-mon[86348]: from='client.64945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:20.583 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:20 smithi059 ceph-mon[81151]: from='client.64945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:20.686 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:20 smithi064 ceph-mon[65271]: from='client.64945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:21.755 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:21 smithi059 ceph-mon[86348]: pgmap v407: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:21.755 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:21 smithi059 ceph-mon[86348]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:32:21.755 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:21 smithi059 ceph-mon[81151]: pgmap v407: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:21.755 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:21 smithi059 ceph-mon[81151]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:32:21.936 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:21 smithi064 ceph-mon[65271]: pgmap v407: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:21.937 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:21 smithi064 ceph-mon[65271]: from='client.64786 -' entity='client.iscsi.foo.smithi059.cazpww' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-26T16:32:23.209 INFO:teuthology.orchestra.run.smithi059.stdout: "v16.2.2", 2024-04-26T16:32:23.210 INFO:teuthology.orchestra.run.smithi059.stdout: "v16.2.2-20210505", 2024-04-26T16:32:23.531 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:23 smithi059 ceph-mon[81151]: pgmap v408: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:23.531 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:23 smithi059 ceph-mon[81151]: from='client.64951 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:23.532 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:23 smithi059 ceph-mon[86348]: pgmap v408: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:23.532 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:23 smithi059 ceph-mon[86348]: from='client.64951 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:23.828 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-26T16:32:23.839 INFO:tasks.cephadm:Teardown begin 2024-04-26T16:32:23.839 DEBUG:teuthology.orchestra.run.smithi059:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T16:32:23.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T16:32:23.897 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:23 smithi064 ceph-mon[65271]: pgmap v408: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-26T16:32:23.897 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:23 smithi064 ceph-mon[65271]: from='client.64951 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T16:32:23.903 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-26T16:32:23.903 DEBUG:teuthology.orchestra.run.smithi059:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T16:32:23.927 DEBUG:teuthology.orchestra.run.smithi064:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T16:32:23.961 INFO:tasks.cephadm:Stopping all daemons... 2024-04-26T16:32:23.962 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-26T16:32:23.962 DEBUG:teuthology.orchestra.run.smithi059:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a 2024-04-26T16:32:24.082 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:24 smithi059 systemd[1]: Stopping Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:32:24.395 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: ::ffff:172.21.15.64 - - [26/Apr/2024:16:32:24] "GET /metrics HTTP/1.1" 200 38554 "" "Prometheus/2.43.0" 2024-04-26T16:32:24.395 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:32:24] ENGINE Bus STOPPING 2024-04-26T16:32:24.395 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:32:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T16:32:24.396 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:32:24] ENGINE Bus STOPPED 2024-04-26T16:32:24.396 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:32:24] ENGINE Bus STARTING 2024-04-26T16:32:24.396 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: CherryPy Checker: 2024-04-26T16:32:24.396 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: The Application mounted at '' has an empty config. 2024-04-26T16:32:24.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a[81147]: 2024-04-26T16:32:24.269+0000 7f990f0cb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:32:24.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a[81147]: 2024-04-26T16:32:24.269+0000 7f990f0cb700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-26T16:32:24.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:24 smithi059 podman[132289]: 2024-04-26 16:32:24.371367322 +0000 UTC m=+0.213304591 container stop a406f6807e94bb9bf5d911c5a2c8b32003c35d518bef652f1ff16aa50ed98b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-26T16:32:24.396 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:24 smithi059 podman[132289]: 2024-04-26 16:32:24.371857582 +0000 UTC m=+0.213794854 container died a406f6807e94bb9bf5d911c5a2c8b32003c35d518bef652f1ff16aa50ed98b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-26T16:32:24.669 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:32:24.669 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:32:24] ENGINE Serving on http://:::9283 2024-04-26T16:32:24.669 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:24 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: [26/Apr/2024:16:32:24] ENGINE Bus STARTED 2024-04-26T16:32:25.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:24 smithi059 podman[132289]: 2024-04-26 16:32:24.788375083 +0000 UTC m=+0.630312354 container cleanup a406f6807e94bb9bf5d911c5a2c8b32003c35d518bef652f1ff16aa50ed98b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:32:25.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:24 smithi059 bash[132289]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a 2024-04-26T16:32:25.278 DEBUG:teuthology.orchestra.run.smithi059:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a.service' 2024-04-26T16:32:25.299 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:25 smithi059 podman[132314]: 2024-04-26 16:32:25.02876436 +0000 UTC m=+0.651566837 container remove a406f6807e94bb9bf5d911c5a2c8b32003c35d518bef652f1ff16aa50ed98b61 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424) 2024-04-26T16:32:25.299 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:25 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T16:32:25.300 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:25 smithi059 systemd[1]: Stopped Ceph mon.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:32:25.300 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 26 16:32:25 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.a.service: Consumed 10.629s CPU time. 2024-04-26T16:32:25.324 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:32:25.324 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-26T16:32:25.325 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-26T16:32:25.325 DEBUG:teuthology.orchestra.run.smithi059:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.c 2024-04-26T16:32:25.583 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:25 smithi059 systemd[1]: Stopping Ceph mon.c for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:32:26.065 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c[86326]: 2024-04-26T16:32:25.674+0000 7f41078e1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:32:26.065 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:25 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c[86326]: 2024-04-26T16:32:25.674+0000 7f41078e1700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-04-26T16:32:26.065 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:25 smithi059 podman[132420]: 2024-04-26 16:32:25.847527555 +0000 UTC m=+0.298930661 container died 0c13fa03c2b8252699e325dfa507d5aa1e19e97e9a84b4e49eac43b2e920f9a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS) 2024-04-26T16:32:26.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:26 smithi059 podman[132420]: 2024-04-26 16:32:26.165377952 +0000 UTC m=+0.616781047 container cleanup 0c13fa03c2b8252699e325dfa507d5aa1e19e97e9a84b4e49eac43b2e920f9a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T16:32:26.333 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 26 16:32:26 smithi059 bash[132420]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-c 2024-04-26T16:32:26.508 DEBUG:teuthology.orchestra.run.smithi059:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.c.service' 2024-04-26T16:32:26.543 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:32:26.543 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-26T16:32:26.544 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-26T16:32:26.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.b 2024-04-26T16:32:26.861 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:26 smithi064 systemd[1]: Stopping Ceph mon.b for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:32:27.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b[65267]: 2024-04-26T16:32:26.860+0000 7f7766779700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:32:27.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b[65267]: 2024-04-26T16:32:26.860+0000 7f7766779700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-04-26T16:32:27.146 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:26 smithi064 podman[99304]: 2024-04-26 16:32:26.892265572 +0000 UTC m=+0.149623319 container died 71107a400e689f0ee4c7a9f9d3845b619f6054a47e399ffefcd66a98611e64a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T16:32:27.869 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:27 smithi064 podman[99304]: 2024-04-26 16:32:27.453508018 +0000 UTC m=+0.710865765 container cleanup 71107a400e689f0ee4c7a9f9d3845b619f6054a47e399ffefcd66a98611e64a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:32:27.869 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:27 smithi064 bash[99304]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b 2024-04-26T16:32:27.869 INFO:journalctl@ceph.mon.b.smithi064.stdout:Apr 26 16:32:27 smithi064 podman[99319]: 2024-04-26 16:32:27.646803651 +0000 UTC m=+0.746717958 container remove 71107a400e689f0ee4c7a9f9d3845b619f6054a47e399ffefcd66a98611e64a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mon-b, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-26T16:32:27.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mon.b.service' 2024-04-26T16:32:27.941 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:32:27.941 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-26T16:32:27.942 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-26T16:32:27.942 DEBUG:teuthology.orchestra.run.smithi059:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y 2024-04-26T16:32:28.253 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:27 smithi059 systemd[1]: Stopping Ceph mgr.y for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:32:28.529 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:32:28.252+0000 7fe3074aa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:32:28.529 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:28 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y[60315]: 2024-04-26T16:32:28.252+0000 7fe3074aa700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-26T16:32:28.529 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:28 smithi059 podman[132544]: 2024-04-26 16:32:28.284714365 +0000 UTC m=+0.147317591 container died c5d6af75064d0a51f8d186230fa316ce3df7c281e67f68549e3bfe6bf08f1615 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:32:28.812 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:28 smithi059 podman[132544]: 2024-04-26 16:32:28.620415939 +0000 UTC m=+0.483019164 container cleanup c5d6af75064d0a51f8d186230fa316ce3df7c281e67f68549e3bfe6bf08f1615 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-26T16:32:28.812 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:28 smithi059 bash[132544]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y 2024-04-26T16:32:29.050 DEBUG:teuthology.orchestra.run.smithi059:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-26T16:32:29.071 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:28 smithi059 podman[132559]: 2024-04-26 16:32:28.813236655 +0000 UTC m=+0.522702672 container remove c5d6af75064d0a51f8d186230fa316ce3df7c281e67f68549e3bfe6bf08f1615 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-y, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:32:29.072 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:29 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y.service: Deactivated successfully. 2024-04-26T16:32:29.072 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:29 smithi059 systemd[1]: Stopped Ceph mgr.y for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:32:29.072 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 26 16:32:29 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.y.service: Consumed 1min 19.273s CPU time. 2024-04-26T16:32:29.085 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:32:29.085 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-26T16:32:29.085 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-26T16:32:29.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x 2024-04-26T16:32:29.413 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:32:29 smithi064 systemd[1]: Stopping Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:32:29.686 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:32:29 smithi064 podman[99424]: 2024-04-26 16:32:29.414392212 +0000 UTC m=+0.132532923 container died b30a628a06dcefc2fa466d0ff187d22a26f866f8a18569f399d167b622e80368 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T16:32:30.050 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:32:29 smithi064 podman[99424]: 2024-04-26 16:32:29.849963512 +0000 UTC m=+0.568104225 container cleanup b30a628a06dcefc2fa466d0ff187d22a26f866f8a18569f399d167b622e80368 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0) 2024-04-26T16:32:30.050 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:32:29 smithi064 bash[99424]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x 2024-04-26T16:32:30.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-26T16:32:30.304 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:32:30 smithi064 podman[99439]: 2024-04-26 16:32:30.051347692 +0000 UTC m=+0.637759585 container remove b30a628a06dcefc2fa466d0ff187d22a26f866f8a18569f399d167b622e80368 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-26T16:32:30.304 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:32:30 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T16:32:30.304 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:32:30 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T16:32:30.304 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:32:30 smithi064 systemd[1]: Stopped Ceph mgr.x for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:32:30.304 INFO:journalctl@ceph.mgr.x.smithi064.stdout:Apr 26 16:32:30 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@mgr.x.service: Consumed 24.960s CPU time. 2024-04-26T16:32:30.317 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:32:30.317 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-26T16:32:30.317 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-26T16:32:30.317 DEBUG:teuthology.orchestra.run.smithi059:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.0 2024-04-26T16:32:30.649 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:30 smithi059 systemd[1]: Stopping Ceph osd.0 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:32:31.083 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[94330]: 2024-04-26T16:32:30.648+0000 7fd1f4c5c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:32:31.083 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[94330]: 2024-04-26T16:32:30.648+0000 7fd1f4c5c700 -1 osd.0 160 *** Got signal Terminated *** 2024-04-26T16:32:31.083 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:30 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0[94330]: 2024-04-26T16:32:30.648+0000 7fd1f4c5c700 -1 osd.0 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:32:33.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:32:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:32:33.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:32:33.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:32:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:32:33.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:32:36.417 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:36 smithi059 podman[132665]: 2024-04-26 16:32:36.162841112 +0000 UTC m=+5.640711624 container died 28dcbd084295d4b3767a1988e13add78aeb11bd50f129ce147cf3f0e2a01311b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-26T16:32:36.683 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:36 smithi059 podman[132665]: 2024-04-26 16:32:36.499668675 +0000 UTC m=+5.977539167 container cleanup 28dcbd084295d4b3767a1988e13add78aeb11bd50f129ce147cf3f0e2a01311b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-26T16:32:36.683 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:36 smithi059 bash[132665]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0 2024-04-26T16:32:37.082 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:36 smithi059 podman[132694]: 2024-04-26 16:32:36.684072034 +0000 UTC m=+0.517665927 container remove 28dcbd084295d4b3767a1988e13add78aeb11bd50f129ce147cf3f0e2a01311b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_CLEAN=True) 2024-04-26T16:32:37.429 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:37 smithi059 podman[132789]: 2024-04-26 16:32:37.036276802 +0000 UTC m=+0.018882003 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:32:37.429 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:37 smithi059 podman[132789]: 2024-04-26 16:32:37.151785384 +0000 UTC m=+0.134390630 container create 546cb98de23460ef1ffea1f592d740499c0674451207a9ac5b71dadf879503bb (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-26T16:32:37.429 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:37 smithi059 podman[132789]: 2024-04-26 16:32:37.362524088 +0000 UTC m=+0.345129299 container init 546cb98de23460ef1ffea1f592d740499c0674451207a9ac5b71dadf879503bb (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:32:37.429 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:37 smithi059 podman[132789]: 2024-04-26 16:32:37.365878998 +0000 UTC m=+0.348484198 container start 546cb98de23460ef1ffea1f592d740499c0674451207a9ac5b71dadf879503bb (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:32:37.833 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:37 smithi059 podman[132789]: 2024-04-26 16:32:37.429071721 +0000 UTC m=+0.411676935 container attach 546cb98de23460ef1ffea1f592d740499c0674451207a9ac5b71dadf879503bb (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:32:37.833 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 26 16:32:37 smithi059 podman[132789]: 2024-04-26 16:32:37.679095221 +0000 UTC m=+0.661700421 container died 546cb98de23460ef1ffea1f592d740499c0674451207a9ac5b71dadf879503bb (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T16:32:38.317 DEBUG:teuthology.orchestra.run.smithi059:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.0.service' 2024-04-26T16:32:38.366 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:32:38.366 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-26T16:32:38.366 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-26T16:32:38.366 DEBUG:teuthology.orchestra.run.smithi059:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.1 2024-04-26T16:32:38.582 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:38 smithi059 systemd[1]: Stopping Ceph osd.1 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:32:39.083 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[99250]: 2024-04-26T16:32:38.735+0000 7f1585f7a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:32:39.083 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[99250]: 2024-04-26T16:32:38.735+0000 7f1585f7a700 -1 osd.1 160 *** Got signal Terminated *** 2024-04-26T16:32:39.083 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:38 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1[99250]: 2024-04-26T16:32:38.735+0000 7f1585f7a700 -1 osd.1 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:32:43.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:32:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:32:43.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:32:43.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:32:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:32:43.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:32:43.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:32:43 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:32:43.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:32:44.520 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:44 smithi059 podman[132896]: 2024-04-26 16:32:44.239577098 +0000 UTC m=+5.612138462 container died 7e91e27615212cb6e3c8d1c9abc93828c233f8b56b9d45c02b9211aac09cf13e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:32:44.833 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:44 smithi059 podman[132896]: 2024-04-26 16:32:44.627392108 +0000 UTC m=+5.999953473 container cleanup 7e91e27615212cb6e3c8d1c9abc93828c233f8b56b9d45c02b9211aac09cf13e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424) 2024-04-26T16:32:44.833 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:44 smithi059 bash[132896]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1 2024-04-26T16:32:45.210 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:44 smithi059 podman[132915]: 2024-04-26 16:32:44.870286312 +0000 UTC m=+0.629700999 container remove 7e91e27615212cb6e3c8d1c9abc93828c233f8b56b9d45c02b9211aac09cf13e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5) 2024-04-26T16:32:45.540 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:45 smithi059 podman[133007]: 2024-04-26 16:32:45.113891387 +0000 UTC m=+0.018792037 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:32:45.540 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:45 smithi059 podman[133007]: 2024-04-26 16:32:45.221114816 +0000 UTC m=+0.126015477 container create 28d49b0b302a58ca79c428d15264743f5ea769f80d9fd6a2a718bb0842b2fcd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-26T16:32:45.540 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:45 smithi059 podman[133007]: 2024-04-26 16:32:45.449223222 +0000 UTC m=+0.354123853 container init 28d49b0b302a58ca79c428d15264743f5ea769f80d9fd6a2a718bb0842b2fcd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-26T16:32:45.540 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:45 smithi059 podman[133007]: 2024-04-26 16:32:45.452597407 +0000 UTC m=+0.357498226 container start 28d49b0b302a58ca79c428d15264743f5ea769f80d9fd6a2a718bb0842b2fcd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-26T16:32:45.540 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:45 smithi059 podman[133007]: 2024-04-26 16:32:45.540297574 +0000 UTC m=+0.445198201 container attach 28d49b0b302a58ca79c428d15264743f5ea769f80d9fd6a2a718bb0842b2fcd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T16:32:45.833 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 26 16:32:45 smithi059 podman[133007]: 2024-04-26 16:32:45.789928511 +0000 UTC m=+0.694829134 container died 28d49b0b302a58ca79c428d15264743f5ea769f80d9fd6a2a718bb0842b2fcd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True) 2024-04-26T16:32:46.575 DEBUG:teuthology.orchestra.run.smithi059:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.1.service' 2024-04-26T16:32:46.620 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:32:46.620 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-26T16:32:46.620 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-26T16:32:46.620 DEBUG:teuthology.orchestra.run.smithi059:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.2 2024-04-26T16:32:46.833 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:46 smithi059 systemd[1]: Stopping Ceph osd.2 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:32:47.332 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[106360]: 2024-04-26T16:32:46.954+0000 7f78f75af700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:32:47.333 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[106360]: 2024-04-26T16:32:46.954+0000 7f78f75af700 -1 osd.2 160 *** Got signal Terminated *** 2024-04-26T16:32:47.333 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:46 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2[106360]: 2024-04-26T16:32:46.954+0000 7f78f75af700 -1 osd.2 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:32:52.748 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:52 smithi059 podman[133111]: 2024-04-26 16:32:52.483015836 +0000 UTC m=+5.629272286 container died e0a6538d233161d4bcdee4096403d0560714e3933c65689c07bdc34394a93f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T16:32:53.083 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:52 smithi059 podman[133111]: 2024-04-26 16:32:52.846558258 +0000 UTC m=+5.992814705 container cleanup e0a6538d233161d4bcdee4096403d0560714e3933c65689c07bdc34394a93f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:32:53.083 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:52 smithi059 bash[133111]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2 2024-04-26T16:32:53.432 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:53 smithi059 podman[133128]: 2024-04-26 16:32:53.089321751 +0000 UTC m=+0.603450564 container remove e0a6538d233161d4bcdee4096403d0560714e3933c65689c07bdc34394a93f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-26T16:32:53.767 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:32:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:32:53.558Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:32:53.767 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:32:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:32:53.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:32:53.767 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:32:53 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:32:53.560Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:32:53.767 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:53 smithi059 podman[133217]: 2024-04-26 16:32:53.336040512 +0000 UTC m=+0.019371747 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:32:53.767 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:53 smithi059 podman[133217]: 2024-04-26 16:32:53.440230968 +0000 UTC m=+0.123562188 container create cc631d1d1b834c1d7efea1a8d0187c5cf501d722cb23598873d66853f02dacd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-26T16:32:53.767 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:53 smithi059 podman[133217]: 2024-04-26 16:32:53.667858016 +0000 UTC m=+0.351189242 container init cc631d1d1b834c1d7efea1a8d0187c5cf501d722cb23598873d66853f02dacd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:32:53.767 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:53 smithi059 podman[133217]: 2024-04-26 16:32:53.671438221 +0000 UTC m=+0.354769449 container start cc631d1d1b834c1d7efea1a8d0187c5cf501d722cb23598873d66853f02dacd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2) 2024-04-26T16:32:53.767 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:53 smithi059 podman[133217]: 2024-04-26 16:32:53.767503337 +0000 UTC m=+0.450834572 container attach cc631d1d1b834c1d7efea1a8d0187c5cf501d722cb23598873d66853f02dacd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-26T16:32:54.082 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:53 smithi059 podman[133217]: 2024-04-26 16:32:53.984326167 +0000 UTC m=+0.667657393 container died cc631d1d1b834c1d7efea1a8d0187c5cf501d722cb23598873d66853f02dacd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-26T16:32:54.537 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:54 smithi059 podman[133243]: 2024-04-26 16:32:54.512272875 +0000 UTC m=+0.522846305 container remove cc631d1d1b834c1d7efea1a8d0187c5cf501d722cb23598873d66853f02dacd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5) 2024-04-26T16:32:54.537 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:54 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-26T16:32:54.537 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 26 16:32:54 smithi059 systemd[1]: Stopped Ceph osd.2 for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:32:54.539 DEBUG:teuthology.orchestra.run.smithi059:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.2.service' 2024-04-26T16:32:54.581 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:32:54.581 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-26T16:32:54.581 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-26T16:32:54.581 DEBUG:teuthology.orchestra.run.smithi059:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.3 2024-04-26T16:32:54.833 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:32:54 smithi059 systemd[1]: Stopping Ceph osd.3 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:32:55.333 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:32:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[113501]: 2024-04-26T16:32:54.931+0000 7f480091d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:32:55.333 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:32:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[113501]: 2024-04-26T16:32:54.931+0000 7f480091d700 -1 osd.3 160 *** Got signal Terminated *** 2024-04-26T16:32:55.333 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:32:54 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[113501]: 2024-04-26T16:32:54.931+0000 7f480091d700 -1 osd.3 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:32:58.083 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:32:57 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[113501]: 2024-04-26T16:32:57.592+0000 7f47fcf11700 -1 osd.3 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.507825+0000 front 2024-04-26T16:32:34.507804+0000 (oldest deadline 2024-04-26T16:32:57.407828+0000) 2024-04-26T16:32:58.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:32:57 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:32:57.810+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:32:58.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:32:57 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:32:57.980+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:32:58.833 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:32:58 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[113501]: 2024-04-26T16:32:58.571+0000 7f47fcf11700 -1 osd.3 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.507825+0000 front 2024-04-26T16:32:34.507804+0000 (oldest deadline 2024-04-26T16:32:57.407828+0000) 2024-04-26T16:32:59.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:32:58 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:32:58.790+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:32:59.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:32:59 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:32:59.007+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:00.083 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:32:59 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3[113501]: 2024-04-26T16:32:59.600+0000 7f47fcf11700 -1 osd.3 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.507825+0000 front 2024-04-26T16:32:34.507804+0000 (oldest deadline 2024-04-26T16:32:57.407828+0000) 2024-04-26T16:33:00.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:32:59 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:32:59.774+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:00.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:00 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:00.017+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:00.508 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:00 smithi059 podman[133321]: 2024-04-26 16:33:00.24516984 +0000 UTC m=+5.436173523 container died 0c13cc20260a5f1b489f6ce9fff32d6d6e463124a1f537567eaa16fe285541d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-26T16:33:00.832 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:00 smithi059 podman[133321]: 2024-04-26 16:33:00.698585462 +0000 UTC m=+5.889589153 container cleanup 0c13cc20260a5f1b489f6ce9fff32d6d6e463124a1f537567eaa16fe285541d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-26T16:33:00.833 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:00 smithi059 bash[133321]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3 2024-04-26T16:33:00.989 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:00 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:33:00.716+0000 7fd2520a1700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.854623+0000 front 2024-04-26T16:32:34.854729+0000 (oldest deadline 2024-04-26T16:33:00.154815+0000) 2024-04-26T16:33:00.990 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:00 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:00.767+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:01.310 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:00 smithi059 podman[133338]: 2024-04-26 16:33:00.991415264 +0000 UTC m=+0.744632193 container remove 0c13cc20260a5f1b489f6ce9fff32d6d6e463124a1f537567eaa16fe285541d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-26T16:33:01.413 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:00 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:00.989+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:01.577 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:01 smithi059 podman[133431]: 2024-04-26 16:33:01.21446581 +0000 UTC m=+0.018454148 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:33:01.577 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:01 smithi059 podman[133431]: 2024-04-26 16:33:01.317252928 +0000 UTC m=+0.121241239 container create cd48079ad6a42a59ad13f68d9fe98353103f2073ac5de5d66ebfe57a5fc1868f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T16:33:01.577 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:01 smithi059 podman[133431]: 2024-04-26 16:33:01.502732639 +0000 UTC m=+0.306720953 container init cd48079ad6a42a59ad13f68d9fe98353103f2073ac5de5d66ebfe57a5fc1868f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True) 2024-04-26T16:33:01.577 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:01 smithi059 podman[133431]: 2024-04-26 16:33:01.505800564 +0000 UTC m=+0.309788889 container start cd48079ad6a42a59ad13f68d9fe98353103f2073ac5de5d66ebfe57a5fc1868f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T16:33:01.577 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:01 smithi059 podman[133431]: 2024-04-26 16:33:01.577711507 +0000 UTC m=+0.381699819 container attach cd48079ad6a42a59ad13f68d9fe98353103f2073ac5de5d66ebfe57a5fc1868f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424) 2024-04-26T16:33:01.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:01.413+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:01.833 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:01 smithi059 podman[133431]: 2024-04-26 16:33:01.813676279 +0000 UTC m=+0.617664596 container died cd48079ad6a42a59ad13f68d9fe98353103f2073ac5de5d66ebfe57a5fc1868f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:33:02.025 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:33:01.736+0000 7fd2520a1700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.854623+0000 front 2024-04-26T16:32:34.854729+0000 (oldest deadline 2024-04-26T16:33:00.154815+0000) 2024-04-26T16:33:02.025 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:01 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:01.773+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:02.350 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 26 16:33:02 smithi059 podman[133457]: 2024-04-26 16:33:02.331294384 +0000 UTC m=+0.511933752 container remove cd48079ad6a42a59ad13f68d9fe98353103f2073ac5de5d66ebfe57a5fc1868f (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:33:02.356 DEBUG:teuthology.orchestra.run.smithi059:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.3.service' 2024-04-26T16:33:02.400 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:33:02.400 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-26T16:33:02.400 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-26T16:33:02.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.4 2024-04-26T16:33:02.429 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:02.025+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:02.687 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:02 smithi064 systemd[1]: Stopping Ceph osd.4 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:33:02.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:02.437+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:02.993 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:02.816+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:02.994 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:33:02.735+0000 7fd259ab5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:33:02.994 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:33:02.735+0000 7fd259ab5700 -1 osd.4 160 *** Got signal Terminated *** 2024-04-26T16:33:02.994 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:33:02.735+0000 7fd259ab5700 -1 osd.4 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:33:02.994 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:33:02.759+0000 7fd2520a1700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.854623+0000 front 2024-04-26T16:32:34.854729+0000 (oldest deadline 2024-04-26T16:33:00.154815+0000) 2024-04-26T16:33:03.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:02 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:02.993+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:03.775 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:03.444+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:03.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:03.560Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:33:03.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:03.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:33:03.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:03 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:03.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:33:04.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:03.834+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:04.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:03 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:33:03.775+0000 7fd2520a1700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.854623+0000 front 2024-04-26T16:32:34.854729+0000 (oldest deadline 2024-04-26T16:33:00.154815+0000) 2024-04-26T16:33:04.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:04.003+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:04.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:04.440+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:04.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:04.375Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.59:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.59:8765: connect: connection refused" 2024-04-26T16:33:04.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:04.375Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.59:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.59:8765: connect: connection refused" 2024-04-26T16:33:04.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:04.377Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.59:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.59:8765: connect: connection refused" 2024-04-26T16:33:04.687 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:04.377Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.59:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.59:8765: connect: connection refused" 2024-04-26T16:33:04.688 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:04.378Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.59:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.59:8765: connect: connection refused" 2024-04-26T16:33:05.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:04.796+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:05.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:05 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:05.009+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:05.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:04 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:33:04.763+0000 7fd2520a1700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.854623+0000 front 2024-04-26T16:32:34.854729+0000 (oldest deadline 2024-04-26T16:33:00.154815+0000) 2024-04-26T16:33:05.779 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:05 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:05.466+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:06.042 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:05 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:33:05.780+0000 7fd2520a1700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.854623+0000 front 2024-04-26T16:32:34.854729+0000 (oldest deadline 2024-04-26T16:33:00.154815+0000) 2024-04-26T16:33:06.042 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:05 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:05.843+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:06.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:06.041+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:06.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:06.041+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:06.762 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:06.485+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:07.061 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4[75894]: 2024-04-26T16:33:06.763+0000 7fd2520a1700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.854623+0000 front 2024-04-26T16:32:34.854729+0000 (oldest deadline 2024-04-26T16:33:00.154815+0000) 2024-04-26T16:33:07.062 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:06.824+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:07.062 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:06.824+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:07.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:07.062+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:07.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:07.062+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:07.845 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:07.506+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:08.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:07.844+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:08.188 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:07 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:07.844+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:08.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:08.082+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:08.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:08.082+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:08.458 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:08 smithi064 podman[99559]: 2024-04-26 16:33:08.234002014 +0000 UTC m=+5.630368737 container died a170975ea549f3dbe524f248b1924b76fca3488b2161739ca2ec288e2f740ad5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-26T16:33:08.827 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:08 smithi064 podman[99559]: 2024-04-26 16:33:08.669593821 +0000 UTC m=+6.065960541 container cleanup a170975ea549f3dbe524f248b1924b76fca3488b2161739ca2ec288e2f740ad5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, GIT_CLEAN=True) 2024-04-26T16:33:08.827 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:08 smithi064 bash[99559]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4 2024-04-26T16:33:08.828 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:08.493+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:08.828 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:08.493+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.553593+0000 front 2024-04-26T16:32:43.553582+0000 (oldest deadline 2024-04-26T16:33:08.253455+0000) 2024-04-26T16:33:09.187 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:08 smithi064 podman[99577]: 2024-04-26 16:33:08.879549208 +0000 UTC m=+0.640429308 container remove a170975ea549f3dbe524f248b1924b76fca3488b2161739ca2ec288e2f740ad5 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:33:09.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:08.827+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:09.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:08 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:08.827+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:09.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:09.064+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:09.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:09.064+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:09.598 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:09.508+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:09.599 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:09.508+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.553593+0000 front 2024-04-26T16:32:43.553582+0000 (oldest deadline 2024-04-26T16:33:08.253455+0000) 2024-04-26T16:33:09.599 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:09 smithi064 podman[99680]: 2024-04-26 16:33:09.288759113 +0000 UTC m=+0.091785614 container create 37eaa86cd98493b51f0fa10a218383d91ef2fc7c171ed29e7dcffbdafea15e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T16:33:09.599 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:09 smithi064 podman[99680]: 2024-04-26 16:33:09.215651517 +0000 UTC m=+0.018678045 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:33:09.599 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:09 smithi064 podman[99680]: 2024-04-26 16:33:09.516687493 +0000 UTC m=+0.319714011 container init 37eaa86cd98493b51f0fa10a218383d91ef2fc7c171ed29e7dcffbdafea15e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-26T16:33:09.599 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:09 smithi064 podman[99680]: 2024-04-26 16:33:09.520470831 +0000 UTC m=+0.323497338 container start 37eaa86cd98493b51f0fa10a218383d91ef2fc7c171ed29e7dcffbdafea15e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T16:33:09.599 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:09 smithi064 podman[99680]: 2024-04-26 16:33:09.59951254 +0000 UTC m=+0.402539063 container attach 37eaa86cd98493b51f0fa10a218383d91ef2fc7c171ed29e7dcffbdafea15e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, ceph=True) 2024-04-26T16:33:09.938 INFO:journalctl@ceph.osd.4.smithi064.stdout:Apr 26 16:33:09 smithi064 podman[99680]: 2024-04-26 16:33:09.836003736 +0000 UTC m=+0.639030250 container died 37eaa86cd98493b51f0fa10a218383d91ef2fc7c171ed29e7dcffbdafea15e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-26T16:33:09.938 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:09.811+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:09.939 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:09 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:09.811+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:10.436 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:10.052+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:10.436 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:10.052+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:10.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.4.service' 2024-04-26T16:33:10.549 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:33:10.550 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-26T16:33:10.550 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-26T16:33:10.550 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.5 2024-04-26T16:33:10.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:10.490+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:10.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:10.490+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.553593+0000 front 2024-04-26T16:32:43.553582+0000 (oldest deadline 2024-04-26T16:33:08.253455+0000) 2024-04-26T16:33:10.687 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:10 smithi064 systemd[1]: Stopping Ceph osd.5 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:33:11.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:10.973+0000 7f5b5bfc3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:33:11.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:10.973+0000 7f5b5bfc3700 -1 osd.5 160 *** Got signal Terminated *** 2024-04-26T16:33:11.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:10.973+0000 7f5b5bfc3700 -1 osd.5 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:33:11.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:10.822+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:11.188 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:10 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:10.822+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:11.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:11.063+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:11.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:11.063+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:11.778 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:11.467+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:11.778 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:11.467+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.553593+0000 front 2024-04-26T16:32:43.553582+0000 (oldest deadline 2024-04-26T16:33:08.253455+0000) 2024-04-26T16:33:12.048 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:11.779+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:12.049 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:11 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:11.779+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:12.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:12.048+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:12.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:12.048+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:12.822 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:12.491+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:12.823 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:12.491+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.553593+0000 front 2024-04-26T16:32:43.553582+0000 (oldest deadline 2024-04-26T16:33:08.253455+0000) 2024-04-26T16:33:13.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:12.823+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:13.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:12 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:12.823+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:13.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:13.046+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:13.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:13.046+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:13.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:13.046+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:13.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:13.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:33:13.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:13.563Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:33:13.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:13 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:13.563Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:33:13.868 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:13.498+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:13.869 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:13.498+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.553593+0000 front 2024-04-26T16:32:43.553582+0000 (oldest deadline 2024-04-26T16:33:08.253455+0000) 2024-04-26T16:33:14.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:14.052+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:14.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:14.052+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:14.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:14.052+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:14.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:13.868+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:14.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:13.868+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:14.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:13 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:13.868+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:14.913 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:14.516+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:14.913 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:14.516+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.553593+0000 front 2024-04-26T16:32:43.553582+0000 (oldest deadline 2024-04-26T16:33:08.253455+0000) 2024-04-26T16:33:15.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:15.033+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:15.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:15.033+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:15.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:15.033+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:15.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:14.913+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:15.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:14.913+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:15.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:14 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:14.913+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:15.937 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:15.541+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.752731+0000 front 2024-04-26T16:32:34.752528+0000 (oldest deadline 2024-04-26T16:33:00.652539+0000) 2024-04-26T16:33:15.937 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:15.541+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.553593+0000 front 2024-04-26T16:32:43.553582+0000 (oldest deadline 2024-04-26T16:33:08.253455+0000) 2024-04-26T16:33:15.937 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5[80482]: 2024-04-26T16:33:15.541+0000 7f5b545af700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:48.754402+0000 front 2024-04-26T16:32:48.754390+0000 (oldest deadline 2024-04-26T16:33:14.654309+0000) 2024-04-26T16:33:16.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:16 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:16.017+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:16.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:16 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:16.017+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:16.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:16 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:16.017+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:16.438 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:15.958+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:16.438 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:15.958+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:16.438 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:15 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:15.958+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:16.830 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:16 smithi064 podman[99789]: 2024-04-26 16:33:16.514313216 +0000 UTC m=+5.728559556 container died 833404a87b94e76731f271c0cb0b83696f09b85f70fd06255ef123c03425fdd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2) 2024-04-26T16:33:17.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:16 smithi064 podman[99789]: 2024-04-26 16:33:16.831214932 +0000 UTC m=+6.045461262 container cleanup 833404a87b94e76731f271c0cb0b83696f09b85f70fd06255ef123c03425fdd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-26T16:33:17.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:16 smithi064 bash[99789]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5 2024-04-26T16:33:17.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:17 smithi064 podman[99806]: 2024-04-26 16:33:17.02397901 +0000 UTC m=+0.505392015 container remove 833404a87b94e76731f271c0cb0b83696f09b85f70fd06255ef123c03425fdd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T16:33:17.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:16 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:16.909+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:17.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:16 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:16.909+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:17.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:16 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:16.909+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:17.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:17 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:17.010+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:17.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:17 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:17.010+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:17.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:17 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:17.010+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:17.751 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:17 smithi064 podman[99898]: 2024-04-26 16:33:17.362576144 +0000 UTC m=+0.018444150 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:33:17.751 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:17 smithi064 podman[99898]: 2024-04-26 16:33:17.49150777 +0000 UTC m=+0.147375828 container create 8d3525e7c240b9667dbd7d215514cabd72f9c080f985d2b1fe652bbe360cca6b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T16:33:17.751 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:17 smithi064 podman[99898]: 2024-04-26 16:33:17.676901015 +0000 UTC m=+0.332769028 container init 8d3525e7c240b9667dbd7d215514cabd72f9c080f985d2b1fe652bbe360cca6b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-26T16:33:17.751 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:17 smithi064 podman[99898]: 2024-04-26 16:33:17.680116904 +0000 UTC m=+0.335984919 container start 8d3525e7c240b9667dbd7d215514cabd72f9c080f985d2b1fe652bbe360cca6b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-26T16:33:17.752 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:17 smithi064 podman[99898]: 2024-04-26 16:33:17.751977839 +0000 UTC m=+0.407845857 container attach 8d3525e7c240b9667dbd7d215514cabd72f9c080f985d2b1fe652bbe360cca6b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424) 2024-04-26T16:33:18.187 INFO:journalctl@ceph.osd.5.smithi064.stdout:Apr 26 16:33:17 smithi064 podman[99898]: 2024-04-26 16:33:17.994501631 +0000 UTC m=+0.650369643 container died 8d3525e7c240b9667dbd7d215514cabd72f9c080f985d2b1fe652bbe360cca6b (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T16:33:18.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:17 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:17.927+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:18.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:17 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:17.927+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:18.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:17 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:17.927+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:18.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:17 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:17.974+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:18.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:17 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:17.974+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:18.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:17 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:17.974+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:18.759 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.5.service' 2024-04-26T16:33:18.806 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:33:18.806 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-26T16:33:18.807 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-26T16:33:18.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.6 2024-04-26T16:33:18.996 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:18 smithi064 systemd[1]: Stopping Ceph osd.6 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:33:18.997 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:18.925+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:18.997 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:18.925+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:18.997 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:18.925+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:19.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:19 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:19.092+0000 7f2135b49700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:33:19.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:19 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:19.092+0000 7f2135b49700 -1 osd.6 160 *** Got signal Terminated *** 2024-04-26T16:33:19.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:19 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:19.092+0000 7f2135b49700 -1 osd.6 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:33:19.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:18.996+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:19.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:18.996+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:19.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:18 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:18.996+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:20.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:19 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:19.959+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:20.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:19 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:19.959+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:20.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:19 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:19.959+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:20.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:20 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:20.008+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:20.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:20 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:20.008+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:20.438 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:20 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:20.008+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:21.186 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:20 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:20.922+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:21.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:20 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:20.922+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:21.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:20 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:20.922+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:21.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:21 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:21.032+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:21.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:21 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:21.032+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:21.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:21 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:21.032+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:21.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:21 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:21.032+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:22.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:21 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:21.890+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:22.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:21 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:21.890+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:22.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:21 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:21.890+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:22.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:22.053+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:22.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:22.053+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:22.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:22.053+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:22.188 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:22.053+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:23.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:22.875+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:23.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:22.875+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:23.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:22 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:22.875+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:23.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:23.091+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:23.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:23.091+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:23.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:23.091+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:23.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:23.091+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:23.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:23.562Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:33:23.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:23.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:33:23.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:23 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:23.563Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:33:24.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:23.873+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:35.433762+0000 front 2024-04-26T16:32:35.433773+0000 (oldest deadline 2024-04-26T16:32:57.733668+0000) 2024-04-26T16:33:24.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:23.873+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:41.234286+0000 front 2024-04-26T16:32:41.234271+0000 (oldest deadline 2024-04-26T16:33:06.534211+0000) 2024-04-26T16:33:24.187 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:23 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6[85491]: 2024-04-26T16:33:23.874+0000 7f212e135700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.235390+0000 front 2024-04-26T16:32:51.235470+0000 (oldest deadline 2024-04-26T16:33:12.935212+0000) 2024-04-26T16:33:24.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:24.052+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:24.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:24.052+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:24.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:24.052+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:24.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:24 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:24.052+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:24.800 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:24 smithi064 podman[100002]: 2024-04-26 16:33:24.479114594 +0000 UTC m=+5.477032497 container died ab006709251fb65dfbf4933d2d2d0e96ab8bc30ffb47c3735b4f2241b9b4206e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2) 2024-04-26T16:33:25.074 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:24 smithi064 podman[100002]: 2024-04-26 16:33:24.800645694 +0000 UTC m=+5.798563594 container cleanup ab006709251fb65dfbf4933d2d2d0e96ab8bc30ffb47c3735b4f2241b9b4206e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T16:33:25.074 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:24 smithi064 bash[100002]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6 2024-04-26T16:33:25.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:25 smithi064 podman[100020]: 2024-04-26 16:33:25.110404534 +0000 UTC m=+0.624903769 container remove ab006709251fb65dfbf4933d2d2d0e96ab8bc30ffb47c3735b4f2241b9b4206e (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424) 2024-04-26T16:33:25.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:25.073+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:25.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:25.073+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:25.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:25.073+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:25.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:25 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:25.073+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:25.937 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:25 smithi064 podman[100114]: 2024-04-26 16:33:25.488304974 +0000 UTC m=+0.018837926 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:33:25.937 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:25 smithi064 podman[100114]: 2024-04-26 16:33:25.586166673 +0000 UTC m=+0.116699607 container create 3705b79ad17e2f29329338b74edfa55cc855cd3bc835c889f59ff172d2f74d55 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-26T16:33:25.937 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:25 smithi064 podman[100114]: 2024-04-26 16:33:25.747058825 +0000 UTC m=+0.277591762 container init 3705b79ad17e2f29329338b74edfa55cc855cd3bc835c889f59ff172d2f74d55 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:33:25.937 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:25 smithi064 podman[100114]: 2024-04-26 16:33:25.750111189 +0000 UTC m=+0.280644123 container start 3705b79ad17e2f29329338b74edfa55cc855cd3bc835c889f59ff172d2f74d55 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-26T16:33:25.937 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:25 smithi064 podman[100114]: 2024-04-26 16:33:25.82179998 +0000 UTC m=+0.352332926 container attach 3705b79ad17e2f29329338b74edfa55cc855cd3bc835c889f59ff172d2f74d55 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-26T16:33:26.437 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:26 smithi064 podman[100114]: 2024-04-26 16:33:26.052927258 +0000 UTC m=+0.583460197 container died 3705b79ad17e2f29329338b74edfa55cc855cd3bc835c889f59ff172d2f74d55 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e) 2024-04-26T16:33:26.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:26.038+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:26.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:26.038+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:26.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:26.038+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:26.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:26 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:26.038+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:26.727 INFO:journalctl@ceph.osd.6.smithi064.stdout:Apr 26 16:33:26 smithi064 podman[100140]: 2024-04-26 16:33:26.708456472 +0000 UTC m=+0.650687337 container remove 3705b79ad17e2f29329338b74edfa55cc855cd3bc835c889f59ff172d2f74d55 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T16:33:26.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.6.service' 2024-04-26T16:33:26.777 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:33:26.777 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-26T16:33:26.777 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-26T16:33:26.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.7 2024-04-26T16:33:27.080 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:26 smithi064 systemd[1]: Stopping Ceph osd.7 for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:33:27.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:27 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:27.079+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:27.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:27 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:27.079+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:27.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:27 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:27.079+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:27.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:27 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:27.079+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:27.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:27 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:27.109+0000 7fe0d1c0e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T16:33:27.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:27 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:27.109+0000 7fe0d1c0e700 -1 osd.7 160 *** Got signal Terminated *** 2024-04-26T16:33:27.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:27 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:27.109+0000 7fe0d1c0e700 -1 osd.7 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T16:33:28.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:28.123+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:28.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:28.123+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:28.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:28.123+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:28.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:28 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:28.123+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:29.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:29.103+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:29.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:29.103+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:29.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:29.103+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:29.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:29 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:29.103+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:30.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:30 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:30.147+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:30.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:30 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:30.147+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:30.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:30 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:30.147+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:30.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:30 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:30.147+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:31.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:31.156+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6806 osd.0 since back 2024-04-26T16:32:34.241162+0000 front 2024-04-26T16:32:34.241220+0000 (oldest deadline 2024-04-26T16:32:57.141010+0000) 2024-04-26T16:33:31.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:31.157+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6814 osd.1 since back 2024-04-26T16:32:43.042049+0000 front 2024-04-26T16:32:43.042026+0000 (oldest deadline 2024-04-26T16:33:05.341706+0000) 2024-04-26T16:33:31.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:31.157+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6822 osd.2 since back 2024-04-26T16:32:51.743197+0000 front 2024-04-26T16:32:51.743182+0000 (oldest deadline 2024-04-26T16:33:12.843092+0000) 2024-04-26T16:33:31.437 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:31 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7[90214]: 2024-04-26T16:33:31.157+0000 7fe0ca1fa700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.59:6830 osd.3 since back 2024-04-26T16:32:55.044492+0000 front 2024-04-26T16:32:55.044551+0000 (oldest deadline 2024-04-26T16:33:20.344334+0000) 2024-04-26T16:33:32.895 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:32 smithi064 podman[100219]: 2024-04-26 16:33:32.503507185 +0000 UTC m=+5.499138268 container died 5757a13fa26cf0fc5c6290f1cb326f7d82ec5e69d3b2771df8a5412e2424eb45 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-26T16:33:33.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:32 smithi064 podman[100219]: 2024-04-26 16:33:32.994270081 +0000 UTC m=+5.989901157 container cleanup 5757a13fa26cf0fc5c6290f1cb326f7d82ec5e69d3b2771df8a5412e2424eb45 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-26T16:33:33.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:32 smithi064 bash[100219]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7 2024-04-26T16:33:33.570 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:33 smithi064 podman[100236]: 2024-04-26 16:33:33.237324541 +0000 UTC m=+0.725575807 container remove 5757a13fa26cf0fc5c6290f1cb326f7d82ec5e69d3b2771df8a5412e2424eb45 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True) 2024-04-26T16:33:33.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:33.562Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:33:33.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:33.563Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi059:8443/api/prometheus_receiver\": dial tcp 172.21.15.59:8443: connect: connection refused" 2024-04-26T16:33:33.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:33 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:33.563Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2024-04-26T16:33:33.872 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:33 smithi064 podman[100328]: 2024-04-26 16:33:33.571494405 +0000 UTC m=+0.113924815 container create c37bb76e5dc2f9ed4cedb5582c206f0cf8e581333fa3921b1f4ec847bb76ca97 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-26T16:33:33.872 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:33 smithi064 podman[100328]: 2024-04-26 16:33:33.475814001 +0000 UTC m=+0.018244389 image pull 031ba2747cfdd41e6bd5399209a1ba77775f760ed8b698251b3c2529efe3552f quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45 2024-04-26T16:33:33.873 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:33 smithi064 podman[100328]: 2024-04-26 16:33:33.790277353 +0000 UTC m=+0.332707722 container init c37bb76e5dc2f9ed4cedb5582c206f0cf8e581333fa3921b1f4ec847bb76ca97 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-26T16:33:33.873 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:33 smithi064 podman[100328]: 2024-04-26 16:33:33.793414816 +0000 UTC m=+0.335845188 container start c37bb76e5dc2f9ed4cedb5582c206f0cf8e581333fa3921b1f4ec847bb76ca97 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-04-26T16:33:33.873 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:33 smithi064 podman[100328]: 2024-04-26 16:33:33.873534125 +0000 UTC m=+0.415964510 container attach c37bb76e5dc2f9ed4cedb5582c206f0cf8e581333fa3921b1f4ec847bb76ca97 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-26T16:33:34.187 INFO:journalctl@ceph.osd.7.smithi064.stdout:Apr 26 16:33:34 smithi064 podman[100328]: 2024-04-26 16:33:34.094140083 +0000 UTC m=+0.636570452 container died c37bb76e5dc2f9ed4cedb5582c206f0cf8e581333fa3921b1f4ec847bb76ca97 (image=quay.ceph.io/ceph-ci/ceph@sha256:4b71a575c225eecd96353b1d64f11c7118b9683e4e8491c513d9f702bd9a1f45, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-phlogistonjohn-testing-2024-04-25-1915-a430f9e, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-26T16:33:34.863 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@osd.7.service' 2024-04-26T16:33:34.907 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:33:34.907 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-26T16:33:34.907 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-26T16:33:34.907 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a 2024-04-26T16:33:35.187 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:34 smithi064 systemd[1]: Stopping Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:33:35.522 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.272Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T16:33:35.522 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.272Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T16:33:35.522 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.272Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T16:33:35.522 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.272Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T16:33:35.522 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.272Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T16:33:35.522 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.272Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T16:33:35.522 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.272Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T16:33:35.523 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.272Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T16:33:35.523 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.272Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T16:33:35.523 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.273Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T16:33:35.523 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.273Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T16:33:35.523 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a[61436]: ts=2024-04-26T16:33:35.273Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T16:33:35.523 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 podman[100434]: 2024-04-26 16:33:35.282302476 +0000 UTC m=+0.141746889 container died 279642636cd094659590087aff8c222754ebe100c627d655eb7ff7c428b97573 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:33:35.797 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 podman[100434]: 2024-04-26 16:33:35.621806577 +0000 UTC m=+0.481251004 container cleanup 279642636cd094659590087aff8c222754ebe100c627d655eb7ff7c428b97573 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:33:35.797 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 bash[100434]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T16:33:36.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-26T16:33:36.068 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:35 smithi064 podman[100446]: 2024-04-26 16:33:35.797678867 +0000 UTC m=+0.513370118 container remove 279642636cd094659590087aff8c222754ebe100c627d655eb7ff7c428b97573 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T16:33:36.068 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:36 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T16:33:36.068 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:36 smithi064 systemd[1]: Stopped Ceph prometheus.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:33:36.068 INFO:journalctl@ceph.prometheus.a.smithi064.stdout:Apr 26 16:33:36 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 4.446s CPU time. 2024-04-26T16:33:36.080 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T16:33:36.081 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-26T16:33:36.081 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b6607c98-03e4-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T16:33:37.909 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:33:37 smithi059 systemd[1]: Stopping Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:33:38.266 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:33:37 smithi059 podman[133834]: 2024-04-26 16:33:37.909619709 +0000 UTC m=+0.113214328 container died feeb6104cfe1f3e22eaac074b81b8610bc746c37ea9547da271e0666f5ca684d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:33:38.583 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:33:38 smithi059 podman[133834]: 2024-04-26 16:33:38.4573971 +0000 UTC m=+0.660991764 container cleanup feeb6104cfe1f3e22eaac074b81b8610bc746c37ea9547da271e0666f5ca684d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:33:38.583 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:33:38 smithi059 bash[133834]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T16:33:39.083 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:33:38 smithi059 podman[133848]: 2024-04-26 16:33:38.750349123 +0000 UTC m=+0.840985675 container remove feeb6104cfe1f3e22eaac074b81b8610bc746c37ea9547da271e0666f5ca684d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T16:33:39.083 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:33:38 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T16:33:39.083 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:33:38 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T16:33:39.083 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:33:38 smithi059 systemd[1]: Stopped Ceph node-exporter.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:33:39.083 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 26 16:33:38 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.409s CPU time. 2024-04-26T16:33:39.461 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:39 smithi059 systemd[1]: Stopping Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:33:39.770 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:39 smithi059 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a[74729]: ts=2024-04-26T16:33:39.461Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T16:33:39.770 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:39 smithi059 podman[133955]: 2024-04-26 16:33:39.502659021 +0000 UTC m=+0.266739318 container died 70f9197770531f034200b9e1e1679b3a0366c72875d79a7a6ef76e0f8a7edbb7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:33:40.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:39 smithi059 podman[133955]: 2024-04-26 16:33:39.861785968 +0000 UTC m=+0.625866252 container cleanup 70f9197770531f034200b9e1e1679b3a0366c72875d79a7a6ef76e0f8a7edbb7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:33:40.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:39 smithi059 bash[133955]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T16:33:40.451 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:40 smithi059 podman[133969]: 2024-04-26 16:33:40.188360429 +0000 UTC m=+0.680439180 container remove 70f9197770531f034200b9e1e1679b3a0366c72875d79a7a6ef76e0f8a7edbb7 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T16:33:40.451 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:40 smithi059 podman[133969]: 2024-04-26 16:33:40.247393665 +0000 UTC m=+0.739472374 volume remove aadaeb58a9677a48a3634a227e0dbff73c4a0af32f3ea5dd5961f24b122fab4a 2024-04-26T16:33:40.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:40 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T16:33:40.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:40 smithi059 systemd[1]: Stopped Ceph alertmanager.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:33:40.833 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 26 16:33:40 smithi059 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.768s CPU time. 2024-04-26T16:34:03.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b6607c98-03e4-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T16:34:05.242 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:34:04 smithi064 systemd[1]: Stopping Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:34:05.564 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:34:05 smithi064 podman[100832]: 2024-04-26 16:34:05.242748495 +0000 UTC m=+0.110966320 container died 2d4920f76a2e228c984cb4c9ebe311fefbe0db3e205bcac88d852647b6fa2eff (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:34:05.564 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:34:05 smithi064 podman[100832]: 2024-04-26 16:34:05.564880385 +0000 UTC m=+0.433098204 container cleanup 2d4920f76a2e228c984cb4c9ebe311fefbe0db3e205bcac88d852647b6fa2eff (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:34:05.836 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:34:05 smithi064 bash[100832]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T16:34:05.836 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:34:05 smithi064 podman[100846]: 2024-04-26 16:34:05.757739464 +0000 UTC m=+0.519124956 container remove 2d4920f76a2e228c984cb4c9ebe311fefbe0db3e205bcac88d852647b6fa2eff (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T16:34:05.836 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:34:05 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T16:34:06.187 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:34:06 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T16:34:06.187 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:34:06 smithi064 systemd[1]: Stopped Ceph node-exporter.b for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:34:06.187 INFO:journalctl@ceph.node-exporter.b.smithi064.stdout:Apr 26 16:34:06 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 3.336s CPU time. 2024-04-26T16:34:06.460 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:06 smithi064 systemd[1]: Stopping Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968... 2024-04-26T16:34:06.727 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=server t=2024-04-26T16:34:06.460985511Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-04-26T16:34:06.728 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=ticker t=2024-04-26T16:34:06.461332303Z level=info msg=stopped last_tick=2024-04-26T16:34:00Z 2024-04-26T16:34:06.728 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:06 smithi064 ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a[63670]: logger=tracing t=2024-04-26T16:34:06.461389393Z level=info msg="Closing tracing" 2024-04-26T16:34:06.728 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:06 smithi064 podman[100967]: 2024-04-26 16:34:06.471368826 +0000 UTC m=+0.108954318 container died 5d3c77d009e2d42a9b99de0e32eaa72421ab5e89fd773f2fb2d8770c5dd453f3 (image=quay.io/ceph/grafana:9.4.12, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-26T16:34:06.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:06 smithi064 podman[100967]: 2024-04-26 16:34:06.810744858 +0000 UTC m=+0.448330359 container cleanup 5d3c77d009e2d42a9b99de0e32eaa72421ab5e89fd773f2fb2d8770c5dd453f3 (image=quay.io/ceph/grafana:9.4.12, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-26T16:34:06.986 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:06 smithi064 bash[100967]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a 2024-04-26T16:34:07.251 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:06 smithi064 podman[100981]: 2024-04-26 16:34:06.98682582 +0000 UTC m=+0.515972034 container remove 5d3c77d009e2d42a9b99de0e32eaa72421ab5e89fd773f2fb2d8770c5dd453f3 (image=quay.io/ceph/grafana:9.4.12, name=ceph-b6607c98-03e4-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-26T16:34:07.251 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:07 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-26T16:34:07.251 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:07 smithi064 systemd[1]: Stopped Ceph grafana.a for b6607c98-03e4-11ef-bc93-c7b262605968. 2024-04-26T16:34:07.251 INFO:journalctl@ceph.grafana.a.smithi064.stdout:Apr 26 16:34:07 smithi064 systemd[1]: ceph-b6607c98-03e4-11ef-bc93-c7b262605968@grafana.a.service: Consumed 12.145s CPU time. 2024-04-26T16:34:19.009 DEBUG:teuthology.orchestra.run.smithi059:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T16:34:19.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T16:34:19.070 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-26T16:34:19.072 DEBUG:teuthology.misc:Transferring archived files from smithi059:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817/remote/smithi059/crash 2024-04-26T16:34:19.073 DEBUG:teuthology.orchestra.run.smithi059:> sudo tar c -f - -C /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/crash -- . 2024-04-26T16:34:19.117 INFO:teuthology.orchestra.run.smithi059.stderr:tar: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T16:34:19.117 INFO:teuthology.orchestra.run.smithi059.stderr:tar: Error is not recoverable: exiting now 2024-04-26T16:34:19.119 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817/remote/smithi064/crash 2024-04-26T16:34:19.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/crash -- . 2024-04-26T16:34:19.156 INFO:teuthology.orchestra.run.smithi064.stderr:tar: /var/lib/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T16:34:19.156 INFO:teuthology.orchestra.run.smithi064.stderr:tar: Error is not recoverable: exiting now 2024-04-26T16:34:19.157 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-26T16:34:19.157 DEBUG:teuthology.orchestra.run.smithi059:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-26T16:34:19.189 INFO:teuthology.orchestra.run.smithi059.stdout:2024-04-26T16:14:37.691011+0000 mon.a (mon.0) 1059 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:34:19.190 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-26T16:34:19.190 DEBUG:teuthology.orchestra.run.smithi059:> sudo egrep '\[SEC\]' /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-26T16:34:19.259 DEBUG:teuthology.orchestra.run.smithi059:> sudo egrep '\[ERR\]' /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-26T16:34:19.327 DEBUG:teuthology.orchestra.run.smithi059:> sudo egrep '\[WRN\]' /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-26T16:34:19.397 INFO:teuthology.orchestra.run.smithi059.stdout:2024-04-26T16:14:37.691011+0000 mon.a (mon.0) 1059 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T16:34:19.397 INFO:tasks.cephadm:Compressing logs... 2024-04-26T16:34:19.397 DEBUG:teuthology.orchestra.run.smithi059:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T16:34:19.439 DEBUG:teuthology.orchestra.run.smithi064:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T16:34:19.463 INFO:teuthology.orchestra.run.smithi059.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-26T16:34:19.463 INFO:teuthology.orchestra.run.smithi059.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T16:34:19.463 INFO:teuthology.orchestra.run.smithi059.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.log 2024-04-26T16:34:19.463 INFO:teuthology.orchestra.run.smithi059.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-26T16:34:19.463 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T16:34:19.467 INFO:teuthology.orchestra.run.smithi059.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-26T16:34:19.467 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-26T16:34:19.467 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.2.log: /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-26T16:34:19.467 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-26T16:34:19.467 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-26T16:34:19.470 INFO:teuthology.orchestra.run.smithi064.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T16:34:19.470 INFO:teuthology.orchestra.run.smithi064.stderr:: No such file or directory 2024-04-26T16:34:19.470 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.log 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.audit.log: /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.7.log: /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.5.log: /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.4.log: /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi064.sqspir.log 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi064.sqspir.log: 76.0% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi064.sqspir.log.gz 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr: 89.2% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T16:34:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T16:34:19.472 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.0.log: 94.0% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T16:34:19.472 INFO:teuthology.orchestra.run.smithi059.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi059.dvtzbj.log 2024-04-26T16:34:19.473 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.cephadm.log: 91.4% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T16:34:19.474 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 95.9% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T16:34:19.474 INFO:teuthology.orchestra.run.smithi064.stderr: --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mon.b.log 2024-04-26T16:34:19.477 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mon.c.log: 94.6% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T16:34:19.481 INFO:teuthology.orchestra.run.smithi059.stderr: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T16:34:19.484 INFO:teuthology.orchestra.run.smithi059.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-26T16:34:19.485 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi059.dvtzbj.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T16:34:19.485 INFO:teuthology.orchestra.run.smithi059.stderr: 76.0% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi059.dvtzbj.log.gz 2024-04-26T16:34:19.490 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mon.b.log: 91.0% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T16:34:19.497 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/tcmu-runner.log 2024-04-26T16:34:19.500 INFO:teuthology.orchestra.run.smithi064.stderr: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T16:34:19.505 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.cephadm.log: 97.1% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T16:34:19.505 INFO:teuthology.orchestra.run.smithi059.stderr:gzip -5 --verbose -- /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T16:34:19.506 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/tcmu-runner.log: 87.1% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/tcmu-runner.log.gz 2024-04-26T16:34:19.522 INFO:teuthology.orchestra.run.smithi059.stderr:/var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-volume.log: 91.0% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T16:34:19.627 INFO:teuthology.orchestra.run.smithi064.stderr: 89.8% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-26T16:34:20.023 INFO:teuthology.orchestra.run.smithi059.stderr: 89.0% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-26T16:34:20.066 INFO:teuthology.orchestra.run.smithi064.stderr: 92.4% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-26T16:34:20.380 INFO:teuthology.orchestra.run.smithi059.stderr: 92.3% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-26T16:34:21.036 INFO:teuthology.orchestra.run.smithi064.stderr: 93.2% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-26T16:34:21.354 INFO:teuthology.orchestra.run.smithi059.stderr: 93.3% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-26T16:34:21.399 INFO:teuthology.orchestra.run.smithi064.stderr: 93.8% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-26T16:34:21.519 INFO:teuthology.orchestra.run.smithi059.stderr: 93.4% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-26T16:34:21.559 INFO:teuthology.orchestra.run.smithi059.stderr: 93.4% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-26T16:34:21.562 INFO:teuthology.orchestra.run.smithi064.stderr: 93.3% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-26T16:34:21.591 INFO:teuthology.orchestra.run.smithi059.stderr: 91.0% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-26T16:34:21.809 INFO:teuthology.orchestra.run.smithi064.stderr: 93.5% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-26T16:34:21.810 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-04-26T16:34:21.810 INFO:teuthology.orchestra.run.smithi064.stderr:real 0m2.356s 2024-04-26T16:34:21.811 INFO:teuthology.orchestra.run.smithi064.stderr:user 0m8.463s 2024-04-26T16:34:21.811 INFO:teuthology.orchestra.run.smithi064.stderr:sys 0m0.312s 2024-04-26T16:34:21.940 INFO:teuthology.orchestra.run.smithi059.stderr: 93.4% -- replaced with /var/log/ceph/b6607c98-03e4-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-26T16:34:21.942 INFO:teuthology.orchestra.run.smithi059.stderr: 2024-04-26T16:34:21.942 INFO:teuthology.orchestra.run.smithi059.stderr:real 0m2.489s 2024-04-26T16:34:21.942 INFO:teuthology.orchestra.run.smithi059.stderr:user 0m11.754s 2024-04-26T16:34:21.942 INFO:teuthology.orchestra.run.smithi059.stderr:sys 0m0.402s 2024-04-26T16:34:21.942 INFO:tasks.cephadm:Archiving logs... 2024-04-26T16:34:21.943 DEBUG:teuthology.misc:Transferring archived files from smithi059:/var/log/ceph to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817/remote/smithi059/log 2024-04-26T16:34:21.944 DEBUG:teuthology.orchestra.run.smithi059:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T16:34:23.403 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/log/ceph to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817/remote/smithi064/log 2024-04-26T16:34:23.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T16:34:24.597 INFO:tasks.cephadm:Removing cluster... 2024-04-26T16:34:24.598 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b6607c98-03e4-11ef-bc93-c7b262605968 --force 2024-04-26T16:34:24.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b6607c98-03e4-11ef-bc93-c7b262605968 --force 2024-04-26T16:34:25.207 INFO:tasks.cephadm:Removing cephadm ... 2024-04-26T16:34:25.207 DEBUG:teuthology.orchestra.run.smithi059:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T16:34:25.224 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T16:34:25.239 INFO:tasks.cephadm:Teardown complete 2024-04-26T16:34:25.239 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-26T16:34:25.249 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-26T16:34:25.249 DEBUG:teuthology.orchestra.run.smithi059:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T16:34:25.266 DEBUG:teuthology.orchestra.run.smithi064:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T16:34:25.279 INFO:teuthology.orchestra.run.smithi059.stderr:bash: line 1: ntpq: command not found 2024-04-26T16:34:25.283 INFO:teuthology.orchestra.run.smithi059.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T16:34:25.283 INFO:teuthology.orchestra.run.smithi059.stdout:=============================================================================== 2024-04-26T16:34:25.283 INFO:teuthology.orchestra.run.smithi059.stdout:^* hv01.front.sepia.ceph.com 3 6 377 42 +2029us[+1829us] +/- 49ms 2024-04-26T16:34:25.283 INFO:teuthology.orchestra.run.smithi059.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 40 -144us[ -144us] +/- 53ms 2024-04-26T16:34:25.284 INFO:teuthology.orchestra.run.smithi059.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 39 -2918us[-2918us] +/- 67ms 2024-04-26T16:34:25.284 INFO:teuthology.orchestra.run.smithi059.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T16:34:25.293 INFO:teuthology.orchestra.run.smithi064.stderr:bash: line 1: ntpq: command not found 2024-04-26T16:34:25.297 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T16:34:25.297 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2024-04-26T16:34:25.297 INFO:teuthology.orchestra.run.smithi064.stdout:^* hv01.front.sepia.ceph.com 3 6 377 43 +2003us[+1894us] +/- 49ms 2024-04-26T16:34:25.297 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 39 -191us[ -191us] +/- 53ms 2024-04-26T16:34:25.297 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 42 -2951us[-2951us] +/- 67ms 2024-04-26T16:34:25.297 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T16:34:25.298 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-26T16:34:25.308 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-26T16:34:25.309 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-26T16:34:25.317 DEBUG:teuthology.orchestra.run.smithi059:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T16:34:25.415 DEBUG:teuthology.orchestra.run.smithi064:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T16:34:25.490 DEBUG:teuthology.orchestra.run.smithi059:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-26T16:34:25.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T16:34:25.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-26T16:34:25.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T16:34:25.547 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-26T16:34:25.556 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-26T16:34:25.564 INFO:teuthology.task.internal:Duration was 2850.080229 seconds 2024-04-26T16:34:25.565 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-26T16:34:25.573 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-26T16:34:25.573 DEBUG:teuthology.orchestra.run.smithi059:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T16:34:25.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T16:34:25.612 INFO:teuthology.orchestra.run.smithi059.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T16:34:25.625 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T16:34:26.122 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-26T16:34:26.122 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi059.front.sepia.ceph.com 2024-04-26T16:34:26.123 DEBUG:teuthology.orchestra.run.smithi059:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T16:34:26.185 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi064.front.sepia.ceph.com 2024-04-26T16:34:26.185 DEBUG:teuthology.orchestra.run.smithi064:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T16:34:26.223 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-26T16:34:26.224 DEBUG:teuthology.orchestra.run.smithi059:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T16:34:26.227 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T16:34:26.291 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-26T16:34:26.291 DEBUG:teuthology.orchestra.run.smithi059:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T16:34:26.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T16:34:26.981 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-26T16:34:26.990 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-26T16:34:26.991 DEBUG:teuthology.orchestra.run.smithi059:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T16:34:27.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T16:34:27.041 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-26T16:34:27.051 DEBUG:teuthology.orchestra.run.smithi059:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T16:34:27.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T16:34:27.078 INFO:teuthology.orchestra.run.smithi059.stdout:kernel.core_pattern = core 2024-04-26T16:34:27.104 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = core 2024-04-26T16:34:27.142 DEBUG:teuthology.orchestra.run.smithi059:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T16:34:27.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T16:34:27.177 DEBUG:teuthology.orchestra.run.smithi064:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T16:34:27.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T16:34:27.198 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-26T16:34:27.207 INFO:teuthology.task.internal:Transferring archived files... 2024-04-26T16:34:27.208 DEBUG:teuthology.misc:Transferring archived files from smithi059:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817/remote/smithi059 2024-04-26T16:34:27.208 DEBUG:teuthology.orchestra.run.smithi059:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T16:34:27.270 DEBUG:teuthology.misc:Transferring archived files from smithi064:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817/remote/smithi064 2024-04-26T16:34:27.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T16:34:27.310 INFO:teuthology.task.internal:Removing archive directory... 2024-04-26T16:34:27.310 DEBUG:teuthology.orchestra.run.smithi059:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T16:34:27.313 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T16:34:27.357 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-26T16:34:27.369 INFO:teuthology.task.internal:Not uploading archives. 2024-04-26T16:34:27.369 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-26T16:34:27.379 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-26T16:34:27.379 DEBUG:teuthology.orchestra.run.smithi059:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T16:34:27.381 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T16:34:27.394 INFO:teuthology.orchestra.run.smithi059.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 16:34 /home/ubuntu/cephtest 2024-04-26T16:34:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 16:34 /home/ubuntu/cephtest 2024-04-26T16:34:27.413 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-26T16:34:27.423 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-26T16:34:27.443 INFO:teuthology.nuke:Checking targets against current locks 2024-04-26T16:34:27.470 DEBUG:teuthology.nuke:shortname: smithi059 2024-04-26T16:34:27.470 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T16:34:27.493 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi059.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 15:41:24.415640', 'locked_by': 'scheduled_phlogistonjohn@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFVJcwblfPLN9tgjmFFh+3o/rieQnpkT9ya3MoVuf0keS/HFhkWfmKmHD2GMT1ipPS8ldbV/6QvNluD1QKAUlG0='} 2024-04-26T16:34:27.496 DEBUG:teuthology.nuke:shortname: smithi064 2024-04-26T16:34:27.496 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T16:34:27.519 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi064.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/phlogistonjohn-2024-04-26_15:06:39-orch:cephadm-wip-phlogistonjohn-testing-2024-04-25-1915-distro-default-smithi/7674817', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 15:41:24.417563', 'locked_by': 'scheduled_phlogistonjohn@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP2vydMHmlTV/e4Mdcnz1q5GRy4z/5gVzfYCuSx3mMdEwaP2Y9RqoqKPZCloEUWumDfo1JvnMWLg48JG/C3hgwM='} 2024-04-26T16:34:27.547 INFO:teuthology.orchestra.console.smithi059:Power off 2024-04-26T16:34:27.547 DEBUG:teuthology.orchestra.console.smithi059:pexpect command: ipmitool -H smithi059.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T16:34:27.571 DEBUG:teuthology.orchestra.console.smithi059:power off output: Chassis Power Control: Down/Off 2024-04-26T16:34:27.571 DEBUG:teuthology.orchestra.console.smithi059:pexpect command: ipmitool -H smithi059.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T16:34:27.589 INFO:teuthology.orchestra.console.smithi064:Power off 2024-04-26T16:34:27.589 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T16:34:27.598 DEBUG:teuthology.orchestra.console.smithi059:check power output: Chassis Power is on 2024-04-26T16:34:27.610 DEBUG:teuthology.orchestra.console.smithi064:power off output: Chassis Power Control: Down/Off 2024-04-26T16:34:27.610 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T16:34:27.632 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is on 2024-04-26T16:34:31.599 DEBUG:teuthology.orchestra.console.smithi059:pexpect command: ipmitool -H smithi059.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T16:34:31.633 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T16:34:31.710 DEBUG:teuthology.orchestra.console.smithi059:check power output: Chassis Power is on 2024-04-26T16:34:31.743 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is on 2024-04-26T16:34:35.714 DEBUG:teuthology.orchestra.console.smithi059:pexpect command: ipmitool -H smithi059.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T16:34:35.744 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T16:34:35.824 DEBUG:teuthology.orchestra.console.smithi059:check power output: Chassis Power is on 2024-04-26T16:34:35.853 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is off 2024-04-26T16:34:35.954 INFO:teuthology.orchestra.console.smithi064:Power off completed 2024-04-26T16:34:39.826 DEBUG:teuthology.orchestra.console.smithi059:pexpect command: ipmitool -H smithi059.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T16:34:39.936 DEBUG:teuthology.orchestra.console.smithi059:check power output: Chassis Power is off 2024-04-26T16:34:40.036 INFO:teuthology.orchestra.console.smithi059:Power off completed 2024-04-26T16:34:40.140 INFO:teuthology.run:Summary data: description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2850.0802285671234 failure_reason: '"2024-04-26T16:14:37.691011+0000 mon.a (mon.0) 1059 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_phlogistonjohn@teuthology success: false 2024-04-26T16:34:40.140 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T16:34:40.230 INFO:teuthology.run:FAIL